generate new image

logs-to-tsdb/backfill/do-all.sh

@@ -2,7 +2,7 @@
 for fil in `ls *.txt`
 do
   echo "ON FILE "$fil
-  python3 munge2.py $fil
+  yes | python3 munge2.py $fil
   rm logs*gz
   rm ready*json
 done

logs-to-tsdb/backfill/get-all-days.py

@@ -36,16 +36,16 @@ def get_query_results(query_id, filename):
 def main():
     log_group = "/ecs/ceramic-prod-cas"
 
-    start_date_str = "2024-06-17 23:59:00"
-    end_date_str = "2024-06-26 00:00:00"
+    start_date_str = "2024-07-01 23:59:00"
+    end_date_str = "2024-07-15 20:00:00"
     start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
     end_date = datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
 
     query_ids = []
 
     # Start all the queries
     while start_date < end_date:
-        next_date = start_date + timedelta(minutes=30)
+        next_date = start_date + timedelta(minutes=5)
         query_id = start_query_for_period(log_group, start_date, next_date)
         if query_id:
             query_ids.append((query_id, start_date.strftime('%Y-%m-%d-%H-%M')))  # Storing the date for filename

logs-to-tsdb/backfill/munge2.py

@@ -70,15 +70,16 @@
 LAM_TEMP = { "awslogs": { "data": "" }}
 
 for logs in ls:
+    print("Inserting data from logstream: " + logs)
     (grp,part,logid) = logs.split('/')
-    fname = 'logs_{}.json'.format(logid)
+    fname = '{}.json'.format(backfile)
     with open(fname, 'w') as f:
         json.dump(ls[logs], f)
     os.system('gzip {}'.format(fname))
     newl = LAM_TEMP.copy()
     newl['awslogs']['data'] = subprocess.run(['base64', '{}.gz'.format(fname)], capture_output=True, text=True).stdout.strip()
 
-    with open('ready_{}.json'.format(logid), 'w') as f:
+    with open('ready_{}.json'.format(backfile), 'w') as f:
        json.dump(newl, f)
-    cmd = 'python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(logid)
-    os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(logid)) 
+    cmd = 'python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(backfile)
+    os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(backfile)) 

logs-to-tsdb/kinesis-lambda.py

@@ -111,3 +111,4 @@ def handler(event, context):
     context = {}
 
     handler(event, context)
+    print("Done")