serious progress

DiskUtils.py

@@ -3,25 +3,54 @@ import time
 import re
 import fnmatch
 
-def dowork():
+def check_download_exists_matching_url_version_one(url):
 
-    ################################################################################################
+    print ("enter > downloads > exist check > version 1")
 
-    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
-    print("checking disk for the stuff...")
+    if not match:
 
-    print("url", url)
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
 
-    fileExists = check_download_exists_matching_url(url)
+        return
 
-    print("file already exists / ",fileExists)
+    channel_name, year, month = match.groups()
 
-    ################################################################################################
+    print("channel / ",channel_name)
 
-def check_download_exists_matching_url(url):
+    #print("year / ",year)
 
-    print ("hello?")
+    #print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
+
+        updated = filename.lower()
+
+        #print("file in downloads / ",updated)
+
+        if fnmatch.fnmatch(updated, file_pattern):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
+
+        return False
+
+def check_download_exists_matching_url_version_two(url):
+
+    print ("enter > check_download_exists_matching_url_version_two")
 
     match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
@@ -39,31 +68,69 @@ def check_download_exists_matching_url(url):
 
     print("month / ",month)
 
-    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
-
-    ## this is a problem. it works but if there 2 channels. 
-    ## redbull and redbull2 this is throw false positives. ugh
-
-    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
-
-    for filename in os.listdir("/root/Downloads/"):
+    for filename in os.listdir("/home/yankee/Downloads/"):
 
         updated = filename.lower()
 
-        print("file in downloads / ",updated)
+        print("found file (v2) / ",updated)
 
-        if fnmatch.fnmatch(updated, file_pattern):
+        ##
 
-            print(f"Found matching file: {updated}")
+        hasKeyDateParts = does_filename_contain_key_date_parts(updated, month, year)
 
-            return True
+        print("\thasKeyDateParts / ",hasKeyDateParts)
 
-    else:
+        ##
 
-        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+        hasTargetChannelName = does_filename_contain_target_channel_name(updated,channel_name)
+
+        print("\thasTargetChannelName / ",hasTargetChannelName)
+
+        ##
+
+    print ("exit < check_download_exists_matching_url_version_two")
+
+def does_filename_contain_target_channel_name(url,channel):  
+
+    regex = rf"\({re.escape(channel)}\)"
+
+    match = re.search(regex, url)
+
+    results = bool(match)
+
+    print("results / ",results)
+
+
+    match = re.search("\({channel}\)", url)
+
+    if match:
+        print("yup!")
+        extracted_text = match.group(1).lower()
+
+        print("extracted: /",extracted_text)
 
         return False
 
-def shutDownRobot():
+	##        return extracted_text == target_string.lower()
+
+    else:
+        print("nope!")
+        return False
+
+
+
+
+
+
+def does_filename_contain_key_date_parts(filename, monthString, yearNumber):
+
+    print("enter -> does_filename_contain_key_date_parts()")
+
+    regex = rf"{re.escape(monthString)} {yearNumber}"
+
+    doesContain = bool(re.search(regex, filename, re.IGNORECASE))
+
+    print("doesContain / ",doesContain)
+
+    return doesContain
 
-    print("shutting down the robot now")

RestAPIClient.py

@@ -79,9 +79,9 @@ def addDownloadFailure(url, error):
 
 def searchURLSuccesses(url):
 
-    print("enter -> callGET()")
+    ## print("enter -> callGET()")
 
-    print("param / url / ",url)
+    ## print("param / url / ",url)
     
     data = { "url": url }
 
@@ -91,15 +91,15 @@ def searchURLSuccesses(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/exists", data=json_data, headers=headers)
 
-    print(response.text)
+    ## print(response.text)
 
     return response.text
 
 def searchURLFailures(url):
 
-    print("enter -> callGET()")
+    ## print("enter -> callGET()")
 
-    print("param / url / ",url)
+    ## print("param / url / ",url)
     
     data = { "url": url }
 
@@ -109,6 +109,6 @@ def searchURLFailures(url):
 
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/exists", data=json_data, headers=headers)
 
-    print(response.text)
+    ## print(response.text)
 
     return response.text

SullyGnomeRobot.py

@@ -10,8 +10,6 @@ import RestAPIClient
 import DiskUtils
 from selenium.webdriver import FirefoxOptions
 
-## https://sullygnome.com/channel/kaicenat/365/games"
-
 def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
@@ -28,7 +26,6 @@ def download(url):
 
     print("url / ", driver.current_url)
 
-
     ##########################################
 
     print("stage / start / tblControl_length check")
@@ -87,17 +84,47 @@ def download(url):
 
     print("file download started. now checking if exist before closing")
 
+    counter = 0
+
+    max_iterations = 5
+
     while True:
 
+        print(f"Loop iteration: {counter}")
+
+        counter += 1
+
+        if counter == max_iterations:
+
+            print(f"Loop exited after {max_iterations} iterations.")
+
+            print("shutting down the robot")
+
+            driver.quit()
+
+            print("robot shut down. breaking. and dying.")
+
+            RestAPIClient.addDownloadFailure(url,"file not found on disk")
+
+            print("stage / complete / database updated w/ FAILURE")
+
+            break
+
         print("checking is file exists... [loop]")
 
-        existsAlready = DiskUtils.check_download_exists_matching_url(url)
+        existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
 
-        print("file already exists / ",existsAlready)
+        print("file already exists (v1) / ",existsAlreadyVersionOne)
 
-        if existsAlready:
+        existsAlreadyVersionTwo = DiskUtils.check_download_exists_matching_url_version_two(url)
 
-            print(f"The file matching the url '{url}' exists.")
+        print("file already exists (v2) / ",existsAlreadyVersionTwo)
+
+        time.sleep(40)
+
+        if existsAlreadyVersionOne:
+
+            print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
 
             print("sleeping for 4 secs before shutting down robot")
 
@@ -109,6 +136,10 @@ def download(url):
 
             print("robot shut down. breaking. and dying.")
 
+            RestAPIClient.addDownloadSuccess(url)
+
+            print("stage / complete / database updated w/ success")
+
             break
 
         else:
@@ -123,8 +154,4 @@ def download(url):
 
     ##########################################
 
-    RestAPIClient.addDownloadSuccess(url)
-
-    print("stage / complete / database updated w/ success")
-
     print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -6,16 +6,10 @@ set -x
 
 ##
 
-echo "what up????"
+##cd /yankee-gnome-fire-consumer
 
-####
-
-cd /yankee-gnome-fire-consumer
-
-pwd 
-
-ls
-
-whoami
+echo "##"
+echo "## init > yankee > consumer"
+echo "##"
 
 python3 consumer.py

consumer.py

@@ -3,43 +3,80 @@ import time
 import SullyGnomeRobot
 import RestAPIClient
 
+### start variables ###
+
+jmsQueue="asian-tester"
+
+host = '38.107.226.34'
+
+port = 61613
+
+
+username = 'admin'
+password = 'password'
+
+
+### end variables ###
+
 print("## ")
-print("## starting > JMS consumer (yankee-sully-channels-monthly)")
+print(f"## starting > JMS consumer app!")
 print("## ")
 
+def writeSullyURLToJMS():
+
+    channel="https://sullygnome.com/channel/yaritaiji/2024october/games"
+    
+    print(f"launch -> jms writer -> channel-> {channel}")
+
+    conn = stomp.Connection([('osiris.yankee.embanet.online', 61613)])
+
+    conn.connect('admin', 'password', wait=True)
+
+    try:
+
+        conn.send(body=channel.strip(), destination=f'/queue/{jmsQueue}')
+
+    finally:
+
+        conn.disconnect()
+        
 class MyListener(stomp.ConnectionListener):
 
     def on_message(headers, message):
 
-        print("** new message! **")
+        ## print("** new message! **")
 
-        print("message / ", message)
+        ## print("message / ", message)
 
-        print("messagebody / ", message.body)
+        print("JMS url -> ", message.body)
 
-        print("checking the value does or doesnt exist in db")
+        ## print("checking the value does or doesnt exist in db")
 
         alreadyExistsSuccess = RestAPIClient.searchURLSuccesses(message.body)
 
-        print("alreadyExistsSuccess / ", alreadyExistsSuccess)
+        ## print("alreadyExistsSuccess / ", alreadyExistsSuccess)
 
         if alreadyExistsSuccess.lower().strip() == "true":
     
-            print("this record exists (successes). returning. doing nothing")
+            ## print("this record exists (successes). returning. doing nothing")
 
-            return
+            ## return
 
-        print("didn't exist in success. check in errors now.")
+            print ("skipped!!")
+            
+        ## print("didn't exist in success. check in errors now.")
 
         alreadyExistsFailure = RestAPIClient.searchURLFailures(message.body)
 
-        print("alreadyExistsFailure / ", alreadyExistsFailure)
+        ## print("alreadyExistsFailure / ", alreadyExistsFailure)
 
         if alreadyExistsFailure.lower().strip() == "true":
 
-            print("this record exists (failures). returning. doing nothing")
+            ## print("this record exists (failures). returning. doing nothing")
 
-            return
+            ##return
+            
+            print ("skipped!!")
 
         print("didn't exist in success or errors. will process now")
 
@@ -60,19 +97,24 @@ class MyListener(stomp.ConnectionListener):
         RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
 
 print("## ")
-print("## starting > JMS consumer (yankee-sully-channels-monthly)")
+print(f"## starting > JMS consumer ([{jmsQueue}]")
 print("## ")
 
-conn = stomp.Connection([('67.220.70.106', 61613)])
+writeSullyURLToJMS()
+connection_details = [(host, port)]
+
+conn = stomp.Connection(connection_details)
 
 conn.set_listener('', MyListener())
 
-conn.connect('admin', 'password', wait=True)
+conn.connect(username, password, wait=True)
 
-conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
+conn.subscribe(f'/queue/{jmsQueue}', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
 
+print(f"Subscribed to queue: {jmsQueue}")
+    
 print("## ")
-print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
+print(f"## notification > connected to JMS server [{jmsQueue}] (upgrade)")
 print("## ")
 
 while True:
@@ -80,5 +122,6 @@ while True:
     pass
 
     time.sleep(1)
-    
+       
 conn.disconnect()
+

recycle/crontabX2

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+48 20 * * *  /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+
+* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
+

recycle/smart-cron-builder.py

@@ -28,7 +28,9 @@ def buildLogFileName():
 
         print("server_hostname / ",server_hostname)
 
-        node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+        #node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        node_id= "/var/log/yankee.log"
 
         return node_id
 
@@ -40,21 +42,27 @@ def buildCronExecuteTime():
 
     current_time = datetime.datetime.now()
 
-    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+    new_minute = (current_time.minute + 2) % 60
 
     print("new_minute / ",new_minute)
 
-    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+    new_hour = (current_time.hour + 5) % 24
 
     print("new_hour / ",new_hour)
 
-    cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
 
     return cron
 
 def buildCronSituation():
 
-    return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+    ##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+
+    ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
 
 ####
 
@@ -103,16 +111,10 @@ def writeCrontabSituation():
 
     file_path ="crontabX"
 
-    crontab_text = """
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
-
-* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
-
     try:
-        # Check if the file exists and delete if it does
+
         if os.path.exists(file_path):
+
             os.remove(file_path)
 
         with open(file_path, "w") as f:
@@ -134,9 +136,9 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
             f.write(sep)
             f.write(sep)
 
-            f.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
-            f.write(sep)
-            f.write(sep)
+            #worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
+            #f.write(sep)
+            #f.write(sep)
 
         print(f"Crontab written to '{file_path}' successfully.")
 
@@ -150,6 +152,6 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 ####################
 
-createLogDir()
+#createLogDir()
 
 writeCrontabSituation()