ec2_maintainer/main.py at main · SurajBhari/ec2_maintainer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import json
from discord_webhook import DiscordWebhook, DiscordEmbed
import requests
from botocore.config import Config
import boto3
import time

def get_second():
    return int(str(int(time.time()))[-1]) # see this logic ? its so complex. yet so simple

logo_link = "https://raw.githubusercontent.com/SurajBhari/ec2_maintainer/main/256_ec2_maintainer.png"
project_name = "Ec2 Maintainer"
count = 1
last_second = get_second()

continous_down = {} # store continous down instances and the count
config = json.load(open('config.json', "r"))

discord_message_dict = {} # store discord messages so that we dont spam the maintainer with messages
# notify all of the discord that the maintainer is starting
for instance in config.keys():
    if not config[instance]['discord']:
        continue
    embed = DiscordEmbed(
        title=f"{instance} is starting!",
        color=0x00ab00,
        description=f"{instance} is starting! Checking every {config[instance]['interval']} seconds..."
    )
    webhook = DiscordWebhook(
        username = project_name,
        avatar_url=logo_link,
        url=config[instance]['discord'],
        embeds=[embed]
    )
    response = webhook.execute()

while True:
    while last_second == get_second():
        time.sleep(0.1)
    last_second = get_second()
    config = json.load(open('config.json', "r"))
    for instance in config.keys():
        if count % config[instance]['interval'] != 0:
            print(f"{count}. Not time to check {instance}...")
            continue # Skip if not time to check
        print(f"Checking {instance}...")
        location = config[instance]['location']
        url = location['url']
        headers = location['headers']
        tolerance = config[instance]['tolerance']
        timeout = location['timeout']
        ok = False
        while tolerance > 0:
            time.sleep(1)
            try:
                response = requests.get(url, headers=headers, timeout=timeout)
            except Exception as e:
                print(f"{instance} is down!")
                tolerance -= 1
                continue
            if response.status_code == location["response_code"]:
                print(f"{instance} Responded with correct code!")
                if instance in discord_message_dict:
                    webhook = discord_message_dict[instance]
                else:
                    webhook = DiscordWebhook(
                        username = project_name,
                        avatar_url=logo_link,
                        url=config[instance]['discord'],
                    )
                webhook.content = f"<t:{int(time.time())}:R> | {instance} is up! Response code: {response.status_code} "
                if instance in discord_message_dict:
                    webhook.edit() # if we already have a message, edit it
                else:
                    webhook.execute()
                webhook.execute()
                discord_message_dict[instance] = webhook
                ok = True
                break
            print(f"{instance} said {response.status_code}!")
            tolerance -= 1
        if continous_down.get(instance, 0) >= config[instance]['max_down']:
            continue # stop caring if its been down for so long so that i can actually do stuff and fix it
        if ok:
            continous_down[instance] = 0
            continue
        if config[instance]['discord']:
            embed = DiscordEmbed(
                title=f"{url} is down!",
                color=0xf54242,
                description=f"{instance} is down! Restarting now..."
            )
            webhook = DiscordWebhook(
                username = project_name,
                avatar_url=logo_link,
                url=config[instance]['discord'],
                embeds=[embed]
            )
            response = webhook.execute()
            if instance in discord_message_dict:
                del discord_message_dict[instance] # remove the message so that we can send a new one
        continous_down[instance] = continous_down.get(instance, 0) + 1
        ec2 = config[instance]['ec2']
        ec2 = boto3.client(
            'ec2',
            region_name=ec2['region'],
            aws_access_key_id=ec2['aws_access_key_id'],
            aws_secret_access_key=ec2['aws_secret_access_key']
        )
        ec2.stop_instances(InstanceIds=[instance])
        waiter = ec2.get_waiter('instance_stopped')
        waiter.wait(InstanceIds=[instance])
        ec2.start_instances(InstanceIds=[instance])
        waiter = ec2.get_waiter('instance_running')
        waiter.wait(InstanceIds=[instance])
        if config[instance]['discord']:
            embed = DiscordEmbed(
                title=f"{instance} has been restarted!",
                color=0x00ab00,
            )
            webhook = DiscordWebhook(
                username = project_name,
                avatar_url=logo_link,
                url=config[instance]['discord'],
                embeds=[embed]
            )
            response = webhook.execute()

    count += 1