add sleeps back

DiskUtils.py

@@ -3,25 +3,54 @@ import time
 import re
 import fnmatch
 
-def dowork():
+def check_download_exists_matching_url_version_one(url):
 
-    ################################################################################################
+    print ("enter > downloads > exist check > version 1")
 
-    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
-    print("checking disk for the stuff...")
+    if not match:
 
-    print("url", url)
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
 
-    fileExists = check_download_exists_matching_url(url)
+        return
 
-    print("file already exists / ",fileExists)
+    channel_name, year, month = match.groups()
 
-    ################################################################################################
+    print("channel / ",channel_name)
 
-def check_download_exists_matching_url(url):
+    #print("year / ",year)
 
-    print ("hello?")
+    #print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
+
+        updated = filename.lower()
+
+        #print("file in downloads / ",updated)
+
+        if fnmatch.fnmatch(updated, file_pattern):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
+
+        return False
+
+def check_download_exists_matching_url_version_two(url):
+
+    print ("enter > downloads > exist check > version 2")
 
     match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
@@ -39,14 +68,16 @@ def check_download_exists_matching_url(url):
 
     print("month / ",month)
 
-    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
-
     ## this is a problem. it works but if there 2 channels. 
     ## redbull and redbull2 this is throw false positives. ugh
 
-    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+    unicode_part = r"[\u3040-\u309F\u30A0-\u30FF_]*" # Matches any hiragana, katakana, and underscore
 
-    for filename in os.listdir("/root/Downloads/"):
+    file_pattern = f"{unicode_part} \({channel_name}\) - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    print("regex :",file_pattern)
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
 
         updated = filename.lower()
 
@@ -60,10 +91,7 @@ def check_download_exists_matching_url(url):
 
     else:
 
-        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version t55555wo)")
 
         return False
 
-def shutDownRobot():
-
-    print("shutting down the robot now")

SullyGnomeRobot.py

@@ -10,8 +10,6 @@ import RestAPIClient
 import DiskUtils
 from selenium.webdriver import FirefoxOptions
 
-## https://sullygnome.com/channel/kaicenat/365/games"
-
 def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
@@ -28,7 +26,6 @@ def download(url):
 
     print("url / ", driver.current_url)
 
-
     ##########################################
 
     print("stage / start / tblControl_length check")
@@ -87,17 +84,41 @@ def download(url):
 
     print("file download started. now checking if exist before closing")
 
+    counter = 0
+
+    max_iterations = 5
+
     while True:
 
+        print(f"Loop iteration: {counter}")
+
+        counter += 1
+
+        if counter == max_iterations:
+
+            print(f"Loop exited after {max_iterations} iterations.")
+
+            print("shutting down the robot")
+
+            driver.quit()
+
+            print("robot shut down. breaking. and dying.")
+
+            RestAPIClient.addDownloadFailure(url,"file not found on disk")
+
+            print("stage / complete / database updated w/ FAILURE")
+
+            break
+
         print("checking is file exists... [loop]")
 
-        existsAlready = DiskUtils.check_download_exists_matching_url(url)
+        existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
 
-        print("file already exists / ",existsAlready)
+        print("file already exists (v1) / ",existsAlreadyVersionOne)
 
-        if existsAlready:
+        if existsAlreadyVersionOne:
 
-            print(f"The file matching the url '{url}' exists.")
+            print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
 
             print("sleeping for 4 secs before shutting down robot")
 
@@ -109,6 +130,10 @@ def download(url):
 
             print("robot shut down. breaking. and dying.")
 
+            RestAPIClient.addDownloadSuccess(url)
+
+            print("stage / complete / database updated w/ success")
+
             break
 
         else:
@@ -123,8 +148,4 @@ def download(url):
 
     ##########################################
 
-    RestAPIClient.addDownloadSuccess(url)
-
-    print("stage / complete / database updated w/ success")
-
     print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -6,16 +6,10 @@ set -x
 
 ##
 
-echo "what up????"
+##cd /yankee-gnome-fire-consumer
 
-####
-
-cd /yankee-gnome-fire-consumer
-
-pwd 
-
-ls
-
-whoami
+echo "##"
+echo "## init > yankee > consumer"
+echo "##"
 
 python3 consumer.py

consumer.py

@@ -7,6 +7,8 @@ print("## ")
 print("## starting > JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
+## https://sullygnome.com/channel/yaritaiji/2024october/games
+
 class MyListener(stomp.ConnectionListener):
 
     def on_message(headers, message):
@@ -63,7 +65,7 @@ print("## ")
 print("## starting > JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
-conn = stomp.Connection([('67.220.70.106', 61613)])
+conn = stomp.Connection([('38.107.226.34', 61613)])
 
 conn.set_listener('', MyListener())
 
@@ -72,7 +74,7 @@ conn.connect('admin', 'password', wait=True)
 conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
 
 print("## ")
-print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
+print("## notification > connected to JMS server (yankee-sully-channels-monthly) 456")
 print("## ")
 
 while True:

recycle/crontabX2

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+48 20 * * *  /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+
+* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
+

recycle/smart-cron-builder.py

@@ -28,7 +28,9 @@ def buildLogFileName():
 
         print("server_hostname / ",server_hostname)
 
-        node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+        #node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        node_id= "/var/log/yankee.log"
 
         return node_id
 
@@ -40,21 +42,27 @@ def buildCronExecuteTime():
 
     current_time = datetime.datetime.now()
 
-    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+    new_minute = (current_time.minute + 2) % 60
 
     print("new_minute / ",new_minute)
 
-    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+    new_hour = (current_time.hour + 5) % 24
 
     print("new_hour / ",new_hour)
 
-    cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
 
     return cron
 
 def buildCronSituation():
 
-    return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+    ##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+
+    ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
 
 ####
 
@@ -103,16 +111,10 @@ def writeCrontabSituation():
 
     file_path ="crontabX"
 
-    crontab_text = """
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
-
-* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
-
     try:
-        # Check if the file exists and delete if it does
+
         if os.path.exists(file_path):
+
             os.remove(file_path)
 
         with open(file_path, "w") as f:
@@ -134,9 +136,9 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
             f.write(sep)
             f.write(sep)
 
-            f.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
-            f.write(sep)
-            f.write(sep)
+            #worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
+            #f.write(sep)
+            #f.write(sep)
 
         print(f"Crontab written to '{file_path}' successfully.")
 
@@ -150,6 +152,6 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 ####################
 
-createLogDir()
+#createLogDir()
 
 writeCrontabSituation()