1#@!#!123s

: / proc / 3 / root / opt / imh-python / lib / python2.7 / site-packages / cmu_ded / Libs /
Filename : CheckUrl.py
back
import os
import json
import common
import difflib
import requests
import operator


class CheckURL(common.Migration):
    """
    Tailoring this to do the needful.
    """

    def __init__(self, rootdir1='', rootdir2='', ssh_connection=None, logger=None):
        """
        Init of INIT's!
        """
        super(CheckURL, self).__init__(logger=logger)

        # Setup for common.Migration.
        self.rootdir1 = rootdir1
        self.rootdir2 = rootdir2
        self.ssh_connection = ssh_connection

        # Setup for
        self.domains = {}
        self.urls = []
        self.result = {}

    def __str__(self):
        """
        String representation of the object.
        :return: String
        """
        return self.result

    def __repr__(self):
        """
        String reprsentation of the object.
        :return: String.
        """
        return "<CheckURL: rootdir1::%s, rootdir2::%s, domains::%s>" % (self.rootdir1, self.rootdir2, self.domains)

    def main(self):
        """
        Main function that runs everything.
        """
        self.debug("Entering CheckURL.main()")

        self.domains = self.get_domains()
        self.urls = self.create_urls()
        self.test_urls()

        return True

    def get_domains(self):
        """
        Getting the domains.
        :return: A dict of domain to
        """
        self.debug("Entering CheckURL.get_domains()")
        return_dict = {}
        filepath = '/etc/userdatadomains.json'

        for string_exec in self.local_cmd_exec("cat %s" % filepath), self.remote_cmd_exec("cat %s" % filepath):
            if "No such file or directory" in string_exec:
                continue

            contents = json.loads(string_exec)
            for key in contents.keys():
                if key in return_dict.keys():
                    return_dict[key] = ":".join([return_dict[key], contents[key][5].replace(":80", "")])
                else:
                    return_dict[key] = contents[key][5].replace(":80", "")

        return return_dict

    def create_urls(self):
        """
        Creating URL's to test with.
        :return: list of URL's
        """
        self.debug("Entering CheckURL.create_urls()")
        urls = []
        for domain in self.domains:
            if ":" not in self.domains[domain]:
                continue

            for ip in self.domains[domain].split(":"):
                urls.append(URL(domain=domain, ip=ip))

        return urls

    def test_urls(self):
        """
        Doing the needful here.
        :return: Dictionary full of domains and if they matched or not.
                "domain": "Match"
        """
        self.debug("Entering CheckURL.test_urls()")
        result = {}
        # Getting all the HTML for each of the websites.
        for url in self.urls:
            self.debug("Testing the following: " + str(url))
            url.request_html()

        self.urls.sort(key=operator.attrgetter('domain'))
        for url in self.urls:
            if url.domain in result.keys():
                continue

            for x_url in self.list_all_elements(url):
                if url == x_url:
                    result[url.domain] = "Match"
                else:
                    d = difflib.Differ()
                    diff = d.compare(url.html.split('\n'), x_url.html.split('\n'))
                    result[url.domain] = "\n".join(diff)
                    break

        self.result = result
        return result

    def list_all_elements(self, url):
        return_list = []
        for y_url in self.urls:
            if url.domain == y_url.domain:
                return_list.append(y_url)
        return return_list


class URL(object):
    """ class for a specific url. """

    def __init__(self, domain='', url='', ip=None, html=None):
        """
        Setup for everythin.
        :param self:
        :return:
        """
        self.domain = domain
        self.url = url
        self.ip = ip
        self.html = html

    def __str__(self):
        """
        String representation of url
        :return: String
        """
        return "%s with the IP %s" % (self.domain, self.ip)

    def __repr__(self):
        """
        String representation of url.
        :return: repr
        """
        return "<URL: domain::%s, url::%s, ip::%s>" % (self.domain, self.url, self.ip)

    def __eq__(self, rhs):
        """ Just returns the comparison of html. """
        return self.html == rhs.html

    def request_html(self):
        """
        Does a request to the domain name, getting the html.
        :return Sets self.html.
        """
        if not self.url:
            self.build_url()

        try:
            req = requests.get(self.url, headers={'Host': self.domain})
        except requests.ConnectionError as e:
            self.html = "Generic ConnectionError."
            return False
        except requests.TooManyRedirects as e:
            self.html = "Too Many Redirects."
            return False
        except Exception:
            self.html = "General error catcher."
            return False

        if req.status_code == 401:
            self.html = "Website is behind authentication."
        elif req.status_code == 403:
            self.html = "Forbidden page."
        elif req.status_code == 404:
            wordpress = False
            if 'link' in list(req.headers) and 'wp-json' in req.headers['link']:
                wordpress = True
                self.html = "WordPress 404 page from TempURL."
            if not wordpress:
                self.html = "404 in one of the URL's."

        elif req.status_code == 500:
            self.html = "500 response."
        else:
            self.html = req.text

        return True

    def build_url(self):
        """
        Returns and sets the url string for requests.
        :return: The IP surrounded by http://
        """
        result = "http://" + self.ip + "/"
        self.url = result
        return result