Add files via upload

multiprocessing_test.py

@@ -9,7 +9,7 @@ import os
 import sys
 import multiprocessing
 import time
-import magic
+import io
 
 num_cores = multiprocessing.cpu_count()
 print(num_cores)
@@ -34,13 +34,34 @@ def process_files(sourcedir, destdir):
     outfile = ''
     for f in filelist:
         inputfile = f
-        blob = open(f).read()
-        m = magic.open(magic.MAGIC_MIME_ENCODING)
-        m.load()
-        encoding = m.buffer(blob)  # "utf-8" "us-ascii" etc
-        open(f, encoding=str(encoding))
+        encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
+                    'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
+                    'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
+                    'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865',
+                    'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006',
+                    'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254',
+                    'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004',
+                    'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp',
+                    'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3',
+                    'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4',
+                    'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
+                    'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab',
+                    'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek',
+                    'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154',
+                    'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be',
+                    'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
+        for e in encodings:
+            try:
+                fh = io.open(f, 'r', encoding=e)
+                fh.readlines()
+                fh.seek(0)
+            except UnicodeDecodeError:
+                print('got unicode error with %s , trying different encoding' % e)
+            else:
+                print('opening the file with encoding:  %s ' % e)
+                break 
         print(os.path.basename(f))
-        for lines in filehandle:
+        for lines in fh:
             print(lines)
             outfile = outfile+lines
         fh.close()