From 227da4b8ad0069a78a436fbca6b026257e8e361c Mon Sep 17 00:00:00 2001 From: David Verhasselt Date: Thu, 15 Jun 2017 13:54:42 +0300 Subject: [PATCH 1/3] =?UTF-8?q?Filter=20on=20dupe=20lines=20in=20output,?= =?UTF-8?q?=20don=E2=80=99t=20blackhole=20everything?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t throw away the baby with the bathwater --- lib/docsplit/image_extractor.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/docsplit/image_extractor.rb b/lib/docsplit/image_extractor.rb index b8fa8f1..b5a005b 100755 --- a/lib/docsplit/image_extractor.rb +++ b/lib/docsplit/image_extractor.rb @@ -38,12 +38,12 @@ def convert(pdf, size, format, previous=nil) timeout = 5.minutes.to_i if previous FileUtils.cp(Dir[directory_for(previous) + '/*'], directory) - result = `MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm mogrify #{common} -unsharp 0x0.5+0.75 \"#{directory}/*.#{format}\" 2>/dev/null`.chomp + result = `MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm mogrify #{common} -unsharp 0x0.5+0.75 \"#{directory}/*.#{format}\" 2>&1 | grep -v '^$' | uniq`.chomp raise ExtractionFailed, result if $? != 0 else page_list(pages).each do |page| out_file = ESCAPE[File.join(directory, "#{basename}_#{page}.#{format}")] - cmd = "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm convert +adjoin -define pdf:use-cropbox=true #{common} #{escaped_pdf}[#{page - 1}] #{out_file} 2>/dev/null".chomp + cmd = "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm convert +adjoin -define pdf:use-cropbox=true #{common} #{escaped_pdf}[#{page - 1}] #{out_file} 2>&1 | grep -v '^$' | uniq".chomp result = `#{cmd}`.chomp raise ExtractionFailed, result if $? != 0 end From f403e27b0b295f85ddd9a1a5cb776a58409b37a5 Mon Sep 17 00:00:00 2001 From: David Verhasselt Date: Thu, 15 Jun 2017 13:57:27 +0300 Subject: [PATCH 2/3] Add timeouts to TextExtractor too --- lib/docsplit/text_extractor.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/docsplit/text_extractor.rb b/lib/docsplit/text_extractor.rb index 0e51476..c6ef8a3 100644 --- a/lib/docsplit/text_extractor.rb +++ b/lib/docsplit/text_extractor.rb @@ -61,22 +61,23 @@ def extract_from_ocr(pdf, pages) base_path = File.join(@output, @pdf_name) escaped_pdf = ESCAPE[pdf] psm = @detect_orientation ? "-psm 1" : "" + timeout = 5.minutes.to_i if pages pages.each do |page| tiff = "#{tempdir}/#{@pdf_name}_#{page}.tif" escaped_tiff = ESCAPE[tiff] file = "#{base_path}_#{page}" - run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert -despeckle +adjoin #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf}[#{page - 1}] #{escaped_tiff} 2>&1" - run "tesseract #{escaped_tiff} #{ESCAPE[file]} -l #{@language} #{psm} 2>&1" + run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm convert -despeckle +adjoin #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf}[#{page - 1}] #{escaped_tiff} 2>&1" + run "timeout #{timeout} tesseract #{escaped_tiff} #{ESCAPE[file]} -l #{@language} #{psm} 2>&1" clean_text(file + '.txt') if @clean_ocr FileUtils.remove_entry_secure tiff end else tiff = "#{tempdir}/#{@pdf_name}.tif" escaped_tiff = ESCAPE[tiff] - run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert -despeckle #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf} #{escaped_tiff} 2>&1" + run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm convert -despeckle #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf} #{escaped_tiff} 2>&1" #if the user says don't do orientation detection or the plugin is not installed, set psm to 0 - run "tesseract #{escaped_tiff} #{ESCAPE[base_path]} -l #{@language} #{psm} 2>&1" + run "timeout #{timeout} tesseract #{escaped_tiff} #{ESCAPE[base_path]} -l #{@language} #{psm} 2>&1" clean_text(base_path + '.txt') if @clean_ocr end ensure From 73c2cef089cafd2f20c1b3af4540b5e5ecc80616 Mon Sep 17 00:00:00 2001 From: David Verhasselt Date: Thu, 15 Jun 2017 14:15:13 +0300 Subject: [PATCH 3/3] Add comment --- lib/docsplit/image_extractor.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/docsplit/image_extractor.rb b/lib/docsplit/image_extractor.rb index b5a005b..a55a60c 100755 --- a/lib/docsplit/image_extractor.rb +++ b/lib/docsplit/image_extractor.rb @@ -38,6 +38,9 @@ def convert(pdf, size, format, previous=nil) timeout = 5.minutes.to_i if previous FileUtils.cp(Dir[directory_for(previous) + '/*'], directory) + # We're adding `| grep -v '^$' | uniq` here and below because if a corrupt PDF is parsed, it generates an infinite amount of identical warnings (with blank lines in between). + # By filtering these we avoid memory bloat when the executing process tries to capture stdout. + # See https://github.com/GetSilverfin/silverfin/issues/1998 result = `MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 timeout #{timeout} gm mogrify #{common} -unsharp 0x0.5+0.75 \"#{directory}/*.#{format}\" 2>&1 | grep -v '^$' | uniq`.chomp raise ExtractionFailed, result if $? != 0 else