crontab add

Reviewed-on: https://code.yankee.embanet.online/yankee/yankee-gnome-fire-consumer/pulls/1

RestAPIClient.py

@@ -1,6 +1,7 @@
 import requests
 import json
 import socket
+import netifaces
 
 ## https://www.nylas.com/blog/use-python-requests-module-rest-apis/#how-to-use-python-requests
 ## https://www.abstractapi.com/guides/ip-geolocation/get-ip-address-python
@@ -21,13 +22,25 @@ def ip():
 
         return None
 
+def hostname():
+
+    try:
+
+        return socket.gethostname()
+
+    except Exception as e:
+
+        print(f"Error getting hostname: {e}")
+
+        return "unknown_host"
+
 def addDownloadSuccess(url):
 
     print("enter -> addDownloadSuccess()")
 
     print("param / url / ",url)
     
-    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname()}
 
     json_data = json.dumps(data)
 
@@ -37,20 +50,20 @@ def addDownloadSuccess(url):
 
     headers = {"Content-Type": "application/json"}
 
-
-
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/success/add", data=json_data, headers=headers)
 
     print(response.text)
 
 
-def addDownloadFailure(url):
+def addDownloadFailure(url, error):
 
     print("enter -> addDownloadFailure()")
 
     print("param / url / ",url)
+
+    print("param / error / ",error)
     
-    data = {"url": url, "hostIP": ip(),"ip": ip(),}
+    data = {"url": url, "hostIP": ip(),"ip": ip(),"hostName":hostname(), "reason":error}
 
     json_data = json.dumps(data)
 
@@ -60,50 +73,16 @@ def addDownloadFailure(url):
 
     headers = {"Content-Type": "application/json"}
 
-
-
     response = requests.post("https://apis.yankee.embanet.online/channel/monthly/games/failure/add", data=json_data, headers=headers)
 
     print(response.text)
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-    #print("params", params)
-
-    #response = requests.post("https://osiris.yankee.embanet.online/channel/monthly/games/success/exists", headers=headers, data=params, params=params)
-    
-    #print("response2", response.json())
-
-
 def searchURLSuccesses(url):
 
     print("enter -> callGET()")
 
     print("param / url / ",url)
     
-    params = dict(
-        origin='Chicago,IL',
-        destination='Los+Angeles,CA',
-        waypoints='Joplin,MO|Oklahoma+City,OK',
-        sensor='false',
-        url2=url
-    )
-
-
-    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
-
     data = { "url": url }
 
     headers = {"Content-Type": "application/json"}
@@ -122,17 +101,6 @@ def searchURLFailures(url):
 
     print("param / url / ",url)
     
-    params = dict(
-        origin='Chicago,IL',
-        destination='Los+Angeles,CA',
-        waypoints='Joplin,MO|Oklahoma+City,OK',
-        sensor='false',
-        url2=url
-    )
-
-
-    #data = {"name": "Jane Smith", "email": "janesmith@example.com", "url": url }
-
     data = { "url": url }
 
     headers = {"Content-Type": "application/json"}

SullyGnomeRobot.py

@@ -16,8 +16,8 @@ def download(url):
 
     print(f"launch -> SullyGnomeRobot -> download() -> {url}")
 
-
     opts = FirefoxOptions()
+
     opts.add_argument("--headless")
 
     driver=webdriver.Firefox(options=opts)
@@ -49,7 +49,7 @@ def download(url):
     
         print(f"Element with class name 'tblControl_length' not found: {e}")
         
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url, "Element with class name 'tblControl_length' not found")
 
         driver.quit()
     
@@ -75,7 +75,7 @@ def download(url):
 
         print(f"Element with class name 'TableExportLinkButton' not found: {e}")
 
-        RestAPIClient.addDownloadFailure(url)
+        RestAPIClient.addDownloadFailure(url,"Element with class name 'TableExportLinkButton' not found")
 
         driver.quit()
     

consumer.bash

@@ -3,15 +3,13 @@
 set -e
 
 set -x
+
 ##
 
-echo "what up????"
+echo "init > consumer.bash"
+
+####
 
 cd /yankee-gnome-fire-consumer
 
-pwd 
-
-ls
-
-whoami
 python3 consumer.py

consumer.py

@@ -2,6 +2,8 @@ import stomp
 import time
 import SullyGnomeRobot
 import RestAPIClient
+import os
+import signal
 
 print("## ")
 print("## starting > JMS consumer (yankee-sully-channels-monthly)")
@@ -51,18 +53,19 @@ class MyListener(stomp.ConnectionListener):
 
             print(f"exception / something went wrong when the sully gnome robot was downloading: {e}")
 
-            RestAPIClient.addDownloadFailure(message.body)
+            RestAPIClient.addDownloadFailure(message.body, e)
 
     def on_error(headers, message):
 
         print('received an error "%s"' % message)
         
-        RestAPIClient.addDownloadFailure(message.body)
+        RestAPIClient.addDownloadFailure(message.body, "error from onmessage (consumer.py)")
 
 print("## ")
-print("## starting > JMS consumer (yankee-sully-channels-monthly)")
+print("## starting > (timed) JMS consumer (yankee-sully-channels-monthly)")
 print("## ")
 
+
 conn = stomp.Connection([('67.220.70.106', 61613)])
 
 conn.set_listener('', MyListener())
@@ -75,10 +78,24 @@ print("## ")
 print("## notification > connected to JMS server (yankee-sully-channels-monthly) 3333")
 print("## ")
 
-while True:
+start_time = time.time()
+
+print("start time /",start_time)
+
+counter=0
+
+while time.time() - start_time < 60:
+
+    counter = counter + 1
 
     pass
 
+    print("waiting for 60 seconds of elapsed time: ",counter)
+
     time.sleep(1)
+
+print("it's been 60 seconds. what up though?")
     
 conn.disconnect()
+
+print("disconnected from JMS. i'll holla!")

crontabX

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/yankee.log
+
+* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1
+

ping.bash

@@ -0,0 +1,4 @@
+
+touch /tmp/`date +%Y_%m_%d_%H_%M_%S`.dat
+
+ls -lha /tmp/*dat

recycle/crontabX

@@ -0,0 +1,6 @@
+SHELL=/bin/bash
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+6 11 * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /tmp/root/Downloads/.logs/yankee-node-96.255.165.82-veneno.log 2>&1
+

smart-cron-builder (copy 1).py

@@ -0,0 +1,151 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        node_id= logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+
+    print("new_minute / ",new_minute)
+
+    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * *"
+
+    return cron
+
+def buildCronSituation():
+
+    return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    crontab_text = """
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+* * * * * python3 /yankee-gnome-fire-consumer/consumer.py >> /var/log/yankee-downloader.log 2>&1
+
+* * * * * /tmp/ping2.bash >> /var/log/cron.log 2>&1"""
+
+    try:
+        # Check if the file exists and delete if it does
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /tmp/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+createLogDir()
+
+writeCrontabSituation()

smart-cron-builder.bash

@@ -0,0 +1,17 @@
+reset
+
+clear
+
+python3 smart-cron-builder.py
+
+####
+
+cp crontabX /etc/cron.d/cronfile
+
+# Give execution rights on the cron job
+chmod 0644 /etc/cron.d/cronfile
+
+# Apply the cron job
+crontab -u root /etc/cron.d/cronfile
+
+crontab -l

smart-cron-builder.py

@@ -0,0 +1,157 @@
+import datetime
+import subprocess
+import socket
+import os
+import requests
+
+def buildLogDirectory():
+
+    return "/root/Downloads/.logs/"
+
+def buildLogFileName():
+
+    logFileDir = buildLogDirectory()
+
+    try:
+
+        response = requests.get("https://api.ipify.org?format=json")  # Or another similar service
+
+        response.raise_for_status()  # Check for HTTP errors (4xx or 5xx)
+
+        ip_data = response.json()
+
+        server_ip = ip_data['ip']
+
+        print("server_ip / ",ip_data['ip'])
+
+        server_hostname = socket.gethostname()
+
+        print("server_hostname / ",server_hostname)
+
+        #node_id=logFileDir + "yankee-node-" + server_ip + "-" + server_hostname + ".log"
+
+        node_id= "/var/log/yankee.log"
+
+        return node_id
+
+    except socket.gaierror:
+
+        return "!!error!!"
+
+def buildCronExecuteTime():
+
+    current_time = datetime.datetime.now()
+
+    new_minute = (current_time.minute + 2) % 60  # The % operator performs the wrap-around
+
+    print("new_minute / ",new_minute)
+
+    new_hour = current_time.hour + (current_time.minute + 5) // 60 # Integer division for hour increment
+
+    print("new_hour / ",new_hour)
+
+    cron = str(new_minute) + " " + str(new_hour) + " " + "* * * "
+
+    return cron
+
+def buildCronSituation():
+
+    return "* * * * * /yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() + "/yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName()
+
+    ##return buildCronExecuteTime() +"/yankee-gnome-fire-consumer/consumer.bash >> " + buildLogFileName() + " 2>&1"
+
+    ##return buildCronExecuteTime() +" python3 /yankee-gnome-fire-consumer/consumer.py >> " + buildLogFileName() + " 2>&1"
+
+####
+
+cronTiming = buildCronExecuteTime()
+
+print("cronTiming / ",cronTiming)
+
+##
+
+logger = buildLogFileName();
+
+print("logger / ",logger)
+
+##
+
+cronFile = buildCronSituation();
+
+print()
+print(cronFile)
+print()
+
+
+
+def createLogDir():
+
+    log_dir = buildLogDirectory()
+
+    print("creating log dir",log_dir)
+
+    try:
+        if not os.path.exists(log_dir):  # Check if the directory exists
+            os.makedirs(log_dir)  # Create the directory (and any necessary parent directories)
+            print(f"Directory '{log_dir}' created successfully.")
+        else:
+            print(f"Directory '{log_dir}' already exists.")
+        return True  # Directory exists or was created successfully
+
+    except OSError as e:
+        print(f"Error creating directory '{log_dir}': {e}")
+        return False
+
+##
+
+
+def writeCrontabSituation():
+
+    file_path ="crontabX"
+
+    try:
+
+        if os.path.exists(file_path):
+
+            os.remove(file_path)
+
+        with open(file_path, "w") as f:
+
+            sep="\n"
+
+            f.write("SHELL=/bin/bash")
+            f.write(sep)
+
+            f.write("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+            f.write(sep)
+            f.write(sep)
+
+            f.write(buildCronSituation())
+            f.write(sep)
+            f.write(sep)
+
+            f.write("* * * * * /yankee-gnome-fire-consumer/ping.bash >> /var/log/cron.log 2>&1")
+            f.write(sep)
+            f.write(sep)
+
+            #worksf.write("* * * * * /yankee-gnome-fire-consumer/consumer.bash >> /var/log/consumer.log 2>&1")
+            #f.write(sep)
+            #f.write(sep)
+
+        print(f"Crontab written to '{file_path}' successfully.")
+
+        return True
+
+    except OSError as e:
+
+        print(f"Error writing to file '{file_path}': {e}")
+
+        return False
+
+####################
+
+#createLogDir()
+
+writeCrontabSituation()