После получения json пропадает рандомная его часть
Имеется написанный на python с использованием библиотеки socket модуль который принимает с гитлаб Json, чтобы потом его распарсить в след модуле
При передаче json иногда пропадает рандомная его часть, никак не могу понять в чем загвоздка код парсера
#time
seconds = time.time()
local_time = time.ctime(seconds)
#HDRS
HDRS = 'HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\n\r\n'
HDRS_404 = 'HTTP/1.1 404 OK\r\nContent-Type: text/html; charset=utf-8\r\n\r\n'
# create webserver socket
socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_server.bind(('ip', 8888))
socket_server.listen(356)
socket_server.settimeout(5)
#Start procesing incoming json
def start_my_server():
#def of compare secret key
def load_secret(secret_token, data):
try:
# slack_message_pipe(step=f'I LOAD JSON')
print('load_secret')
key = str(data)
key = re.findall(f'X-Gitlab-Token:...............', key)
print(key)
key = str(key).replace("['X-Gitlab-Token: ", '')
key = str(key).replace("']", '')
print(key)
print(secret_token, ' !!! ', key)
if secret_token == key:
socket_server.settimeout(None)
try_to_verification(key)
else:
fail_verifivcation()
except Exception as e:
print(e)
return
# slack_message_pipe(step=f'start_my_server.load_secret {e}')
def try_to_verification(key):
try:
print(key, 'key try_to_verification')
client_socket.send(HDRS.encode('utf-8'))
client_socket.shutdown(socket.SHUT_WR)
# with open(f"path to file('{local_time}').json", 'w+') as output_file:
# json.dump(data, output_file)
with open(f"path to file", 'w+') as file:
json.dump(data, file)
file.close()
print('next step')
json_dump_for_proj(data)
except Exception as e:
print(e)
return
# slack_message_pipe(step=f'start_my_server.try_to_verification {e}')
def fail_verifivcation():
try:
print('Not find')
client_socket.send(HDRS_404.encode('utf-8'))
client_socket.shutdown(socket.SHUT_WR)
addresses = open('ipPOST', 'a')
addresses.write(str(address) + f'{local_time}\n')
addresses.close()
except Exception as e:
print(e)
return
# slack_message_pipe(step=f'start_my_server.fail_verifivcation {e}')
while True:
print('start loop')
try:
print('try loop')
while True:
print('Working...')
client_socket, address = socket_server.accept()
print('loop', address)
data = client_socket.recv(1048576).decode('utf-8')
# slack_message_pipe(step=f'I GOT JSON')
load_secret(secret_token, data)
except Exception as e:
# slack_message_pipe(step=f'start_my_server.socket.error {e}')
print(f'pass try {e}')
fail_verifivcation()
код распарсивателя
home_path = os.environ['home']
# time
seconds = time.time()
local_time = time.ctime(seconds)
def json_dump_for_proj(data):
os.chdir(home_path)
try:
data = str(data).replace('null', '0')
# Find head json
data = re.sub('POST / HTTP/1.1\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
# data = re.sub(',total_commits_count.*', '}', data)
data = re.sub('POST / HTTP/1.1\r\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
# data = re.sub('.total_commits_count.*', '}', data)
data = re.sub('POST.*\r\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
data = re.sub('"POST / HTTP/1.1\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
data = re.sub('"POST / HTTP/1.1\r\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
data = re.sub('"POST.*\r\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n', '', data)
data = json.loads(data)
# parse needly info
# Branch
# print(data['ref'])
branch = data['ref']
# print(data['commits'])
for keys in data['commits']:
# id
id_hash = keys['id']
# author
# name
name = keys['author']['name']
# email
email = keys['author']['email']
# files
# added
added = keys['added']
for _ in range(len(added) + 1):
for j in added:
if 'path to file' not in j:
added.remove(j)
# print('path to file' not in added[-1])
# modif
modified = keys['modified']
for _ in range(len(modified) + 1):
for k in modified:
if 'force-app/main/default' not in k:
print(k)
modified.remove(k)
print(id_hash, name, email, branch, modified, sep='\n' + '*' * 100 + '\n')
list_of = [(name, email), added, modified, id_hash]
# write_list(list_of)
# print(not modified and not added)
message_dict = {"name": name, "email": email, "modified": modified, "added": added}
if not modified and not added:
slack_message_pipe_good(
step=f' \nI got commit by {message_dict.get("name")}\nEmail: {message_dict.get("email")}\n\nBut it is empty, pass')
return
try:
# slack_message_pipe_good(step=f' \nI got commit by {message_dict.get("name")}\nEmail: {message_dict.get("email")}\n\nInside this commit, the following changes\nadded:{message_dict.get("added")}\nmodified:{message_dict.get("modified")}\n\n I am going to the next step')
git_checkout(id_hash, message_dict)
except Exception as e:
slack_message_pipe(step=f'ERROS ON STEP before_deploy_Parse_Json.json_dump_for_proj: {e}')
return
except Exception as e:
with open(f'data_failure{local_time}.json',
'w+') as data_failure:
data_failure.write(data)
data_failure.close()
slack_message_pipe(step=f' before_deploy_Parse_Json.json_dump_for_proj {e}')
def write_list(list_of):
try:
with open(f'path to file', 'w+') as output_file:
output_file.write(str(list_of) + '\n')
output_file.close()
print('I all write')
except Exception as e:
slack_message_pipe(step=f' before_deploy_Parse_Json.write_list {e}')
# with open('data.json', 'r') as json_file:
# data = json.load(json_file)
# json_dump_for_proj(data)
if __name__ == '__main__':
print('Parse_Json')
UPD: пример json
POST / HTTP/1.1\r\nContent-Type: application/json\r\nUser-Agent: GitLab/15.1.0-pre\r\nX-Gitlab-Event: Push Hook\r\nX-Gitlab-Token: \"X-Gitlab-Token\"\r\nX-Gitlab-Event-Uuid: X-Gitlab-Event-Uuid\r\nConnection: close\r\nHost: nHost:8888\r\nContent-Length: 2736\r\n\r\n{\"object_kind\":\"push\",\"event_name\":\"push\",\"before\":\"event_name\",\"after\":\"event_name\",\"ref\":\"refs/heads/Staging\",\"checkout_sha\":\"checkout_sha\",\"message\":null,\"user_id\":user_id,\"user_name\":\"user_name \",\"user_username\":\"user_name.Sapozhnikov\",\"user_email\":\"\",\"user_avatar\":\"url\",\"project_id\":project_id,\"project\":{\"id\":project_id,\"name\":\"name\",\"description\":\"name repository.\",\"web_url\":\"url\",\"avatar_url\":null,\"git_ssh_url\":\"url\",\"git_http_url\":\"url",\"namespace\":\"Salesforce-Gold\",\"visibility_level\":0,\"path_with_namespace\":\"url\",\"default_branch\":\"master\",\"ci_config_path\":null,\"homepage\":\"url\",\"url\":\"url\",\"ssh_url\":\"url\",\"http_url\":\"url\"},\"commits\":[{\"id\":\"\",\"message\":\"message steps\",\"timestamp\":\"2timestamp\",\"url\":\"url\",\"author\":{\"name\":\"user_name\",\"email\":\"user_name.user_name@user_name.com\"},\"added\":[\"__Release_Notes/Q2.5/CE-2823.docx\",\"__Release_Notes/Q2.5/CE-2823_PROD.csv\",\"__Release_Notes/Q2.5/CE-2823_UAT.csv\"],\"modified\":[\"__Release_Notes/Q2.5.md\"],\"removed\":[]}],\"total_commits_count\":1,\"push_options\":{},\"repository\":{\"name\":\"NA_GOLD\",\"url\":\"url\",\"description\":\"North America Gold Org repository.\",\"homepage\":\"url\",\"git_http_url\":\"url\",\"git_ssh_url\":\"url\",\"visibility_level\":0}}