Import Libraries¶
In [5]:
#!/usr/bin/env python3
from termcolor import colored
Initialise notebook options¶
In [6]:
source_path = "/home/hass/Development/smmtdata-evolved/ocr/"
#source_path = "ocr/"
import_name = source_path + "OUT_3a_custtype_OCR_newformat-2023.txt"
import_file = open(import_name, 'r').readlines()
output_path = "/home/hass/Development/smmtdata-evolved/ocr/"
#output_path = "ocr/"
output_name = output_path + "OUT_3b_custtype_CLEANSE_newformat-2023.csv"
output_file = open(output_name, 'w')
Replacements & removals lists¶
Replacements list tuple
In [ ]:
REPLACEMENTS = [
("MHEV diesel", "MHEV_Diesel"),
("MHEVdiesel", "MHEV_Diesel"),
("MHEV petrol", "MHEV_Petrol"),
("MHEVpetrol", "MHEV_Petrol"),
(" 3 ", " "),
(" 5 ", " "),
(" 2 ", " "),
(" i ", " "),
(" ", " "),
(",", " "),
(" ", ","),
('January','January,,'),
('February','February,,'),
('March','March,,'),
('April','April,,'),
('May','May,,'),
('June','June,,'),
('July','July,,'),
('August','August,,'),
('September','September,,'),
('October','October,,'),
('November','November,,'),
('December','December,,'),
('JANUARY','January,,'),
('FEBRUARY','February,,'),
('MARCH','March,,'),
('APRIL','April,,'),
('MAY','May,,'),
('JUNE','June,,'),
('JULY','July,,'),
('AUGUST','August,,'),
('SEPTEMBER','September,,'),
('OCTOBER','October,,'),
('NOVEMBER','November,,'),
('DECEMBER','December,,'),
('2019,2018',',2019,2018'),
('2019,2018',',2019,2018'),
('2020,2019',',2020,2019'),
('2021,2020',',2021,2020'),
('2022,2021',',2022,2021'),
('2023,2022',',2023,2022'),
(',,20',',20')
]
1 - Clean the data¶
- Remove unwanted change and market share lines
- Correct Tesseract common errors
- Remove blank lines
- Generate a clean CSV for the PANDAS import
- Track line numbers for error reporting
In [ ]:
print("\n")
print("\n")
print("DATA CLEANSING : " + import_name)
uline = colored("================================================================================================================================", "black", "on_white")
print(uline)
print("\n")
line_count = 0 # count lines
for line in import_file:
if "change" in line == True:
pass
# ignore line
# print("Change True" + "\n")
elif "mkt" in line == True:
pass
# also do nothing
# print("mkt True" + "\n")
elif "Mkt" in line == True:
pass
# also do nothing
# print("Mkt True" + "\n")
elif line == "\n":
pass
# also do nothing
# print("Mkt True" + "\n")
elif len(line.strip()) == 0:
pass
# also do nothing / skip line
# print("Blank Line" + "\n")
else:
for old, new in REPLACEMENTS:
line = line.replace(old,new)
linecheck = line.strip().split(",")
if len(linecheck) != 3:
print("================================================================================================================================")
print("Line Error : " + str(line_count) + " Field Count : " + str(len(linecheck)))
error_line = colored(line, "black", "on_white").replace("\n","")
print(error_line)
output_line = line.strip().replace("\n","")
output_file.write(output_line)
print(output_line)
output_file.write("\n")
#print(line)
line_count += 1
output_file.close()
print("The generated file needs to be validated, OCR is not 100% reliable")
print("================================================================================================================================")
print("\n")
print("Import File : " + import_name)
print("\n")
print("Output File : " + output_name)
print("\n")
print("================================================================================================================================")