add sleeps back

Reviewed-on: https://code.yankee.embanet.online/yankee/yankee-gnome-fire-consumer/pulls/1

DiskUtils.py

@@ -3,25 +3,54 @@ import time
 import re
 import fnmatch
 
-def dowork():
+def check_download_exists_matching_url_version_one(url):
 
-    ################################################################################################
+    print ("enter > downloads > exist check > version 1")
 
-    url = "https://sullygnome.com/channel/kaicenat/2023january/streams"
+    match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
-    print("checking disk for the stuff...")
+    if not match:
 
-    print("url", url)
+        print(f"Invalid URL format. Could not extract channel name, year, or month.")
 
-    fileExists = check_download_exists_matching_url(url)
+        return
 
-    print("file already exists / ",fileExists)
+    channel_name, year, month = match.groups()
 
-    ################################################################################################
+    print("channel / ",channel_name)
 
-def check_download_exists_matching_url(url):
+    #print("year / ",year)
 
-    print ("hello?")
+    #print("month / ",month)
+
+    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
+
+    ## this is a problem. it works but if there 2 channels. 
+    ## redbull and redbull2 this is throw false positives. ugh
+
+    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
+
+        updated = filename.lower()
+
+        #print("file in downloads / ",updated)
+
+        if fnmatch.fnmatch(updated, file_pattern):
+
+            print(f"Found matching file: {updated}")
+
+            return True
+
+    else:
+
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version one)")
+
+        return False
+
+def check_download_exists_matching_url_version_two(url):
+
+    print ("enter > downloads > exist check > version 2")
 
     match = re.search(r"https://sullygnome.com/channel/([^/]+)/(\d+)([a-z]+)/games", url.lower())
 
@@ -39,14 +68,16 @@ def check_download_exists_matching_url(url):
 
     print("month / ",month)
 
-    #file_pattern = f"{channel_name}*{month}*{year}*.csv"
-
     ## this is a problem. it works but if there 2 channels. 
     ## redbull and redbull2 this is throw false positives. ugh
 
-    file_pattern = f"*{channel_name} - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+    unicode_part = r"[\u3040-\u309F\u30A0-\u30FF_]*" # Matches any hiragana, katakana, and underscore
 
-    for filename in os.listdir("/root/Downloads/"):
+    file_pattern = f"{unicode_part} \({channel_name}\) - game stats on Twitch in {month} {year} - SullyGnome.csv".lower()
+
+    print("regex :",file_pattern)
+
+    for filename in os.listdir("/home/yankee/Downloads/"):
 
         updated = filename.lower()
 
@@ -60,10 +91,7 @@ def check_download_exists_matching_url(url):
 
     else:
 
-        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'.")
+        print(f"No matching CSV file found for channel '{channel_name}' in year '{year}' and month '{month}'. (version t55555wo)")
 
         return False
 
-def shutDownRobot():
-
-    print("shutting down the robot now")

RestAPIClient.py

@@ -1,6 +1,7 @@
 import requests
 import json
 import socket
+import netifaces
 
 ## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests
 ## https://www.abstractapi.com/guides/ip-geolocation/get-ip-address-python
@@ -21,13 +22,25 @@ def ip():
 
         return None
 
+def hostname():
+
+    try:
+
+        return socket.gethostname()
+
+    except Exception as e:
+
+        print(f"Error getting hostname: {e}")
+
+        return "unknown_host"
+
 def addDownloadSuccess(url):
 
     print("enter -> addDownloadSuccess()")
 
     print("param / url / ",url)
     
-    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname()}
 
     json_data = json.dumps(data)
 
@@ -37,20 +50,20 @@ def addDownloadSuccess(url):
 
     headers = {"Content-Type": "application/json"}
 
-
-
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/add", data=json_data, headers=headers)
 
     print(response.text)
 
 
-def addDownloadFailure(url):
+def addDownloadFailure(url, error):
 
     print("enter -> addDownloadFailure()")
 
     print("param / url / ",url)
 
-    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+    print("param / error / ",error)
+    
+    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname(), "reason":error}
 
     json_data = json.dumps(data)
 
@@ -60,50 +73,16 @@ def addDownloadFailure(url):
 
     headers = {"Content-Type": "application/json"}
 
-
-
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/add", data=json_data, headers=headers)
 
     print(response.text)
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-    #print("params", params)
-
-    #response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", headers=headers, data=params, params=params)
-    
-    #print("response2", response.json())
-
-
 def searchURLSuccesses(url):
 
     print("enter -> callGET()")
 
     print("param / url / ",url)
     
-    params = dict(
-        origin='Chicago,IL',
-        destination='Los+Angeles,CA',
-        waypoints='Joplin,MO|Oklahoma+City,OK',
-        sensor='false',
-        url2=url
-    )
-
-
-    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
-
     data = { "url": url }
 
     headers = {"Content-Type": "application/json"}
@@ -122,17 +101,6 @@ def searchURLFailures(url):
 
     print("param / url / ",url)
     
-    params = dict(
-        origin='Chicago,IL',
-        destination='Los+Angeles,CA',
-        waypoints='Joplin,MO|Oklahoma+City,OK',
-        sensor='false',
-        url2=url
-    )
-
-
-    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
-
     data = { "url": url }
 
     headers = {"Content-Type": "application/json"}

SullyGnomeRobot.py

@@ -10,14 +10,12 @@ import RestAPIClient
 import DiskUtils
 from selenium.webdriver import FirefoxOptions
 
-## https://sullygnome.com/channel/kaicenat/365/games"
-
 def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
 
-
     opts = FirefoxOptions()
+
     opts.add_argument("--headless")
 
     driver=webdriver.Firefox(options=opts)
@@ -28,7 +26,6 @@ def download(url):
 
     print("url / ", driver.current_url)
 
-
     ##########################################
 
     print("stage / start / tblControl_length check")
@@ -49,7 +46,7 @@ def download(url):
     
         print(f"Element with class name 'tblControl_length' not found: {e}")
         
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url, "Element with class name 'tblControl_length' not found")
 
         driver.quit()
     
@@ -75,7 +72,7 @@ def download(url):
 
         print(f"Element with class name 'TableExportLinkButton' not found: {e}")
 
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url,"Element with class name 'TableExportLinkButton' not found")
 
         driver.quit()
     
@@ -87,17 +84,41 @@ def download(url):
 
     print("file download started. now checking if exist before closing")
 
+    counter = 0
+
+    max_iterations = 5
+
     while True:
 
+        print(f"Loop iteration: {counter}")
+
+        counter += 1
+
+        if counter == max_iterations:
+
+            print(f"Loop exited after {max_iterations} iterations.")
+
+            print("shutting down the robot")
+
+            driver.quit()
+
+            print("robot shut down. breaking. and dying.")
+
+            RestAPIClient.addDownloadFailure(url,"file not found on disk")
+
+            print("stage / complete / database updated w/ FAILURE")
+
+            break
+
         print("checking is file exists... [loop]")
 
-        existsAlready = DiskUtils.check_download_exists_matching_url(url)
+        existsAlreadyVersionOne = DiskUtils.check_download_exists_matching_url_version_one(url)
 
-        print("file already exists / ",existsAlready)
+        print("file already exists (v1) / ",existsAlreadyVersionOne)
 
-        if existsAlready:
+        if existsAlreadyVersionOne:
 
-            print(f"The file matching the url '{url}' exists.")
+            print(f"The file matching the url '{url}' DOES INDEED exists. (version 1)")
 
             print("sleeping for 4 secs before shutting down robot")
 
@@ -109,6 +130,10 @@ def download(url):
 
             print("robot shut down. breaking. and dying.")
 
+            RestAPIClient.addDownloadSuccess(url)
+
+            print("stage / complete / database updated w/ success")
+
             break
 
         else:
@@ -123,8 +148,4 @@ def download(url):
 
     ##########################################
 
-    RestAPIClient.addDownloadSuccess(url)
-
-    print("stage / complete / database updated w/ success")
-
     print(f"complete -> SullyGnomeRobot -> download() -> {url}")

consumer.bash

@@ -3,15 +3,13 @@
 set -e
 
 set -x
+
 ##
 
-echo "what up????"
+##cd /yankee-gnome-fire-consumer
 
-cd /yankee-gnome-fire-consumer
+echo "##"
+echo "## init > yankee > consumer"
+echo "##"
 
-pwd 
-
-ls
-
-whoami
 python3 consumer.py

consumer.py

@@ -7,6 +7,8 @@ print("## ")
 print("## starting > JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
+## https://sullygnome.com/channel/yaritaiji/2024october/games
+
 class MyListener(stomp.ConnectionListener):
 
     def on_message(headers, message):
@@ -51,19 +53,19 @@ class MyListener(stomp.ConnectionListener):
 
             print(f"exception / something went wrong when the sully gnome robot was downloading: {e}")
 
-            RestAPIClient.addDownloadFailure(message.body)
+            RestAPIClient.addDownloadFailure(message.body, e)
 
     def on_error(headers, message):
 
         print('received an error "%s"' % message)
         
-        RestAPIClient.addDownloadFailure(message.body)
+        RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
 
 print("## ")
 print("## starting > JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
-conn = stomp.Connection([('67.220.70.106', 61613)])
+conn = stomp.Connection([('38.107.226.34', 61613)])
 
 conn.set_listener('', MyListener())
 
@@ -72,7 +74,7 @@ conn.connect('admin', 'password', wait=True)
 conn.subscribe('/queue/yankee-sully-channels-monthly', id=1, ack='auto', headers={'activemq.prefetchSize': 10})
 
 print("## ")
-print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
+print("## notification > connected to JMS server (yankee-sully-channels-monthly) 456")
 print("## ")
 
 while True:

recycle/crontabX

@@ -0,0 +1,6 @@
+SHELL=/bin/bash
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+6 11 * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /tmp/root/Downloads/.logs/yankee-node-96.255.165.82-veneno.log 2>&1
+

recycle/crontabX2

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+48 20 * * *  /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+
+* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
+

recycle/ping.bash

@@ -0,0 +1,4 @@
+
+touch /tmp/`date +%Y_%m_%d_%H_%M_%S`.dat
+
+ls -lha /tmp/*dat

recycle/smart-cron-builder (copy 1).py

@@ -0,0 +1,151 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+
+    print("new_minute / ",new_minute)
+
+    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
+
+    return cron
+
+def buildCronSituation():
+
+    return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    crontab_text = """
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
+
+* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
+
+    try:
+        # Check if the file exists and delete if it does
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /tmp/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+createLogDir()
+
+writeCrontabSituation()

recycle/smart-cron-builder.bash

@@ -0,0 +1,17 @@
+reset
+
+clear
+
+python3 smart-cron-builder.py
+
+####
+
+cp crontabX /etc/cron.d/cronfile
+
+# Give execution rights on the cron job
+chmod 0644 /etc/cron.d/cronfile
+
+# Apply the cron job
+crontab -u root /etc/cron.d/cronfile
+
+crontab -l

recycle/smart-cron-builder.py

@@ -0,0 +1,157 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        #node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        node_id= "/var/log/yankee.log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60
+
+    print("new_minute / ",new_minute)
+
+    new_hour = (current_time.hour + 5) % 24
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
+
+    return cron
+
+def buildCronSituation():
+
+    ##return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    return buildCronExecuteTime() + " /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() +" /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+
+    ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    try:
+
+        if os.path.exists(file_path):
+
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+            #worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
+            #f.write(sep)
+            #f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+#createLogDir()
+
+writeCrontabSituation()