python

2019-11-08 02:05:43

字体：大中小

来源：转载

供稿：网友

#!/usr/bin/python# _*_ coding: utf-8 _*_# hadooploader csv File checkimport time, datetimeimport socketimport osimport iodef rwfilename(): ###拼接要扫描的csv名称 ###将文件读取到list中保存，待用 ###获取当前时间 ticks = time.time() ###拼接要扫描的csv名称 day = time.strftime("%Y%m%d", time.localtime()) localtime = str(day) hour = time.strftime("%H", time.localtime()) ###小时数减1，表示查询前一小时的文件 hour1 = int(hour) - 1 #PRint(hour1) if hour1<10 : starthour = localtime + "0"+ str(hour1) + "00" else: starthour = localtime + str(hour1) + "00" if int(hour)<10: endhour = localtime +"0"+ str(int(hour)) + "00" else: endhour = localtime + hour + "00" ###定义一个全局的变量传递csv文件名称 global name1 ###最后要扫描的文件名 name1 = ("Hadooploader_" + starthour + "-" + endhour + ".csv") # print(name1) ###调用cfilename(),拿到要扫描的文件名称 ###resultlegth，用于计数器，表示最后有多少行满足条件的总数 resultlegth = 0 ### 获取本机电脑名 global host_name host_name = socket.getfqdn(socket.gethostname()) ### 获取本机的ip global host_ip host_ip = socket.gethostbyname(host_name) # with open('E:/Hadooploader_201611281000-201611281100.csv') as csvfile: ###文件流打开 global fname fname1 = "/home/hadoop-loader/log/record/" + name1 print(fname1) if os.path.exists(fname1): fname = fname1 else: fname = "/home/hadoop-loader/log/data_bak/" + name1 #fname = 'E:/Hadooploader_201612051600-201612051700.csv' ###判断是否有前一小时的csv文件，没有则不进行任何操作 print(fname) global keylist keylist =[] if os.path.exists(fname): file = open(fname) ###逐行读取 #Hadooploader_201612060800-201612060900.csv line = file.readline() ###listcsv用于存取读取到的所有记录的二维列表，子元素为单行的内容 global listcsv listcsv = [] ###用于存储分割前包含filename所在的记录 linestr = [] while line != '': listcsv.append(line.split(",")) line = file.readline() #print(listcsv) ###这个循环用于计数，求得满足条件的行数，创建最后保存结果的二维列表 ###文件流关闭操作。 file.close() resultlegth = 0 for i in range(len(listcsv)): #print(i) ### 取得filename linestr = str(listcsv[i]).split(",") # print(linestr) csv1 = str(linestr[0]) csv2 = csv1[2:len(csv1) - 1] # print(csv2) csv2length = len(csv2) # print("###############") ###取得文件后缀名 endWord = csv2[csv2length - 3:csv2length] if endword == "dat": keylist.append(i) resultlegth = resultlegth + 1 # print(resultlegth) ###最后保存结果的二维列表 #print(keylist) #print(len(keylist)) #print(resultlegth) if len(keylist) > 0: if resultlegth >1000: if (resultlegth/1000)>int(resultlegth/1000) : txtfilenum = int(resultlegth/1000) +1 else: txtfilenum = int(resultlegth/1000) if txtfilenum >16: if (txtfilenum/16)>int(txtfilenum/16): dirnum = int(txtfilenum/16) +1 else: dirnum = int(txtfilenum / 16) else: dirnum =1 else: txtfilenum = 1 dirnum = 1 #print("******************") # print(resultlegth) print(txtfilenum) #print(dirnum) #print("******************") for i in range(resultlegth): ti = int(keylist[i]) linestr = str(listcsv[ti]).split(",") csv1 = str(linestr[0]) csv2 = csv1[2:len(csv1) - 1] # print(csv2) csv2length = len(csv2) csv3 = csv2.split("/") # print(csv3) length1 = len(csv3) csv4 = csv3[length1 - 1] # print(csv4) ###拼接出filename ##filename = csv3[length1 - 4] + "_" + csv3[length1 - 3] + "_" + csv3[length1 - 2] + "_" + csv3[length1 - 1] filename = csv4 # print(filename) ### 取得文件大小 filesizestr = linestr[2] filesize = int(filesizestr[2:len(filesizestr) - 1]) ### 获取clltime clttimestr = linestr[3] clttime = clttimestr[2:len(clttimestr) - 5] clttime1 = clttimestr[2:len(clttimestr) - 1] ctime = datetime.datetime.strptime(clttime, "%Y-%m-%d %H:%M:%S") ctime1 = datetime.datetime.strptime(clttime1, "%Y-%m-%d %H:%M:%S:%f") ### 获取endtime endtimestr = linestr[4] endtime1 = endtimestr[2:len(endtimestr) - 4] endtime = endtimestr[2:len(endtimestr) - 8] etime = datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M:%S") etime1 = datetime.datetime.strptime(endtime1, "%Y-%m-%d %H:%M:%S:%f") duration = int(((etime1 - ctime1).microseconds) / 1000 + ((etime1 - ctime1).seconds) * 1000) #print(filesize) #print(i) #print(duration) dirnum1 = int(i/1000/16) +1 txtfilenum1 = int(i / 1000) if txtfilenum1 % 16 ==0 and txtfilenum1 != 0: txtfilenum2 = 16 else: txtfilenum2 =(txtfilenum1%16) + 1 nowtime = datetime.datetime.now() ###当前小时数 nowh = int(nowtime.strftime("%H")) nowm = int(nowtime.strftime("%M")) + 1 +txtfilenum2 if nowm >= 60: nowm = nowm - 60 nowh = nowh + 1 if int(nowm) < 10: nowm = "0" + str(nowm) if nowh < 10: nowh = "0" + str(nowh) dirname = "/home/vmax-saltagent/sample/HOST_Manager/hadoopload" + str(dirnum1) +"/" txtfilename = "file_integrity_hadooploader_" + str(nowh) + "_" + str(nowm) + ".txt" lastname = dirname + txtfilename if not os.path.exists(dirname): ###判断文件是否存在，返回布尔值 os.mkdir(dirname) if i % 1000 ==0: if not os.path.exists(dirname): ###判断文件是否存在，返回布尔值 os.mkdir(dirname) txtfilename1 = lastname print(i) print(txtfilename1) ###打开文件写入流 ###打开文件写入流 fileopen1 = open(txtfilename1, 'a') ###写入第一行，字段名称 fileopen1.write( "host_name" + "/t" + "host_ip" + "/t" + "module_name" + "/t" + "proc_name" + "/t" + "clttime" + "/t" + "duration" + "/t" + "destip" + "/t" + "filename" + "/t" + "filesize" + "/t" + "gentime" + "/t" + "overtime" + "/t" + "load_result" + "/t" + "error_info" + "/n") ###将结果循环写到生成的txt文件中去 fileopen1.close() fileopen = open(lastname, 'a') fileopen.write(host_name + "/t") fileopen.write(host_ip + "/t") fileopen.write("Hadooploader" + "/t") fileopen.write("hadooploader" + "/t") fileopen.write(endtime + "/t") fileopen.write(str(duration) + "/t") fileopen.write("null" + "/t") fileopen.write(filename + "/t") fileopen.write(str(filesize) + "/t") fileopen.write(endtime + "/t") fileopen.write("0" + "/t") fileopen.write(str(1) + "/t") fileopen.write("upload success" + "/n") fileopen.close()rwfilename()

上一篇：Python模块: collections

下一篇：第一部分 flask简介《Flask Web开发：基于Python的Web应用开发实战》