import json
import pandas as pd
import sys
import numpy as np
import re
class handle_data():
def transfer_to_list(self, string): #转化成列表格式
string = str(string)
string = string.replace(" ", ",")
string = string.replace("\t", ",")
string = string.replace("\n", "")
# string = string.replace("""{\'\\\'""","").replace("""\\\'\'}""","").replace('\\','').replace('''\'''','').replace("""{\'\\\\\\'""","")\
# .replace("""\\\\\\""","")
string = string.replace("""\\\\\\""","").replace("'{","").replace("}'","").replace('""','"')
string = string.split(',')
return string
def transfer_to_str(self, df): #转化成字符串格式
file = ''
for n in range(len(df)):
a = df.iloc[n]
a = str(tuple(a))
a = a.replace('(', '')
a = a.replace(')', '')
a = a.replace('"', '')
a = a.replace("'", "")
a = a.replace(" ","")
if a.find(':') > 0:
try:
a=json.loads(a)
except:
pass
else:
a = a.replace(",","\t")
if n < len(df) - 1:
file = file + a + '\n'
else:
file = file + a
return file
def data_cleaning(self, df, argv1=1, value=999999):
'''
:param df: 读取的数据
:param argv1: 参数如果是1;参数如果是别的;
:param value: 把空的值换成999999;空的值替换成自己的
:return:
'''
file = list(map(handle_data().transfer_to_list, df))
file = pd.DataFrame(file)
try: # 删除重复值
file.drop_duplicates(inplace=True)
except:
pass
if argv1 == 2:
try:
file.replace('', value, inplace=True)
except:
pass
try:
file.fillna(value=value, inplace=True)
except:
pass
file = handle_data().transfer_to_str(file)
return file
def main(self):
try:
df = sys.stdin
if len(sys.argv) == 1&nbs***bsp;len(sys.argv) > 3&nbs***bsp;\
(len(sys.argv) == 2 and int(sys.argv[1]) > 2)&nbs***bsp;\
((len(sys.argv) == 3) and int(sys.argv[1]) > 2):
df = handle_data().data_cleaning(df)
elif len(sys.argv) == 2:
argv1 = int(sys.argv[1])
df = handle_data().data_cleaning(df, argv1)
else:
argv1 = int(sys.argv[1])
value = int(sys.argv[2])
df = handle_data().data_cleaning(df, argv1, value)
# df = handle_data().processing_special_columns(df)
print(df)
except:
pass
if __name__ == '__main__':
handle_data().main()
运行脚本之后数据可以清洗出来但是类型不对,写不进去hive里,报错FAILED: SemanticException [Error 10044]: Line 1:18 Cannot insert into target table because column number/types are different 'waybill_base_info': Cannot convert column 224 from string to map<string,string>.
#Python#