D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
proc
/
2
/
root
/
proc
/
2
/
root
/
opt
/
imh-python
/
lib
/
python2.7
/
site-packages
/
cmu_ded
/
Libs
/
Filename :
cmumain.py
back
Copy
import os import json import socket import shutil import traceback import subprocess from rads.shared import is_cpanel_user from rads.cpanel_api import api_success, whm_api # Local imports. import common import CheckUrl class CMUMain(common.Migration): """ This class basically contains everything that needs to be done regardless if it is shared or dedicated cPanel server. I would like to ( At some point ) get shared -> v/ded working. Anything specific to dedicated servers should be in cmuded.py """ def __init__(self, dev_mode=False, logger=None): """ Setting up logging and shell to be used. :param shell: Shell to be used throughout the process. :param reseller: Reseller user or single user to move. """ super(CMUMain, self).__init__(logger=logger, dev_mode=dev_mode) # Variables needed. self.dev_mode = dev_mode self.str_content = "" self.ssh_connection = None self.base_dir = '/opt/cmu_ded/' # Variables needed between execution. self.users = None self.reseller_user = None self.generated_backups = None self.downloaded_backups = None self.restored_backups = None self.files_downloaded = None self.users_setup = None self.ssls_installed = None # Setup all the things. self.setup() def start_move(self): """ Void function that run's the whole move process. :return: Nothing. """ if self.users is None: try: self.users = self.get_users_to_move() except: self.users = None self.critical(traceback.format_exc()) pass self.dump() if self.generated_backups is None: try: self.generated_backups = self.create_cpanel_backup() except: self.generated_backups = None self.critical(traceback.format_exc()) pass self.dump() # Checking to make sure there are backups to move over. if self.generated_backups: if self.downloaded_backups is None: try: self.downloaded_backups = self.download_backups() except: self.download_backups = None self.critical(traceback.format_exc()) pass self.dump() # We need download_backups to return true before going further to check for 0 byte cpmove files if self.restored_backups is None: try: self.restored_backups = self.restore_cpanel_backup() except: self.restored_backups = None self.critical(traceback.format_exc()) pass self.dump() if self.files_downloaded is None: try: self.files_downloaded = self.download_files() except: self.files_downloaded = None self.critical(traceback.format_exc()) pass self.dump() if self.users_setup is None: try: self.users_setup = self.setup_users() except: self.users_setup = None self.critical(traceback.format_exc()) pass self.dump() if self.ssls_installed is None: try: self.ssls_installed = self.transfer_ssls() except: self.ssls_installed = None self.critical(traceback.format_exc()) pass self.dump() # cPanel license should be valid before we start. self.str_content = self.genstr( users=self.users, gen_backups=self.generated_backups, down_backups=self.downloaded_backups, restored_backups=self.restored_backups, files=self.files_downloaded, setup_users=self.users_setup, ssls=self.ssls_installed ) def setup(self): """ Creating the directory structure here. :return: Creation of the directory structure. /opt/cmu_ded/ tmp/ # Temp files for the move. SSH-Keys and stuff. users/ # users to move. backups/ # Backups that have been moved. files/ # Status of files that have been moved. setup/ # Users that have been setup. finished/ # Finished accounts that have been moved. logs/ # logs for transfers. This should generally be created already. EA # Status of EA transfer. DNS # Status of DNS transfer. PID # PID of main proc. env.setup # Used to validate setup has actually ran. rsync.pid # PID's of RSYNC procs. """ directories = ['tmp', 'users', 'backups', 'files', 'setup', 'finished', 'logs'] for name in directories: if not os.path.exists(self.base_dir + name): os.makedirs(self.base_dir + name) # Touching env.setup. open(self.base_dir + 'env.setup', 'a').close() def dump(self): """ This function should be overloaded. Dumping the variables of the class. This allows --resume to work. :return: dumps the above variables into env.setup as json. """ json_data = { 'users': self.users, 'reseller_user': self.reseller_user, 'generated_backups': self.generated_backups, 'downloaded_backups': self.downloaded_backups, 'restored_backups': self.restored_backups, 'files_downloaded': self.files_downloaded, 'users_setup': self.users_setup, 'ssls_installed': self.ssls_installed } if os.path.exists(self.base_dir + "env.setup"): with open(self.base_dir + "env.setup", 'w') as file: json.dump(json_data, file, sort_keys=True, indent=4) def load(self): """ This function should be overloaded. Loads the data of env.setup. If nothing is in it, Value error will be raised. :return: Sets variables of the class. """ json_data = None if os.path.exists(self.base_dir + "env.setup"): with open(self.base_dir + "env.setup", 'r') as file: try: json_data = json.load(file) except ValueError: self.error("[!!] Doesn't look like cmu_ded was running. Please just run --start.") exit(0) if json_data is not None: self.users = json_data['users'] self.reseller_user = json_data['reseller_user'] self.generated_backups = json_data['generated_backups'] self.downloaded_backups = json_data['downloaded_backups'] self.restored_backups = json_data['restored_backups'] self.files_downloaded = json_data['files_downloaded'] self.users_setup = json_data['users_setup'] self.ssls_installed = json_data['ssls_installed'] def cleanup(self): """ Cleanup script for when the move is complete. :return:This is just removing old files. """ # If there are any error's we don't cleanup. if not self.downloaded_backups or not self.restored_backups or not self.files_downloaded or not self.users_setup: for user in self.get_users(): self.info('Removing /home/%s.old' % user) if os.path.exists('/home/%s.old' % user): shutil.rmtree('/home/%s.old' % user) self.info("(>^.^)> Done <(^.^<)") def genstr(self, **kwargs): """ Basically the splash page for the STR. :return: Splash for STR. """ str_string = "\nSummary of automatic account move from %s to %s\n" % \ (self.ssh_connection.host, socket.gethostname()) str_string += "Also a debug log is located @ logs/cmu.debug\n" str_string += "[1] Users that have been moved: %s\n" % common.to_string(kwargs['users']) str_string += "[2] Backups that have been generated: %s\n" % common.to_string(kwargs['gen_backups']) str_string += "[3] Errors in backups being downloaded: %s\n" % common.to_string(kwargs['down_backups']) str_string += "[4] Errors in restoring backups: %s\n" % common.to_string(kwargs['restored_backups']) str_string += "[5] Errors in downloading user files: %s\n" % common.to_string(kwargs['files']) str_string += "[6] Errors in setting up users: %s\n" % common.to_string(kwargs['setup_users']) str_string += "[7] Errors in installing SSL's: %s\n" % common.to_string(kwargs['ssls']) return str_string def send_str(self): """ Just that, sending the email for extra contacts. :return: Sends an email to str@ """ self.success(self.str_content) self.success("Sending STR.") def get_users_to_move(self): """ Getting users to move to the server we are on. Removes the system user from the list. Then files are created for each user to be moved. This helps with resuming and reseting the program. :return: List of users to transfer. """ self.ssh_connection.check_connection() self.info("Entering get users to move.") output = [] if self.reseller_user.lower() == 'root': # If listdir fails it returns IOError. This will be logged. # Example: IOError: [Errno 2] No such file on /var/cpanel/users/asdasdasd output = self.ssh_connection.sftp.listdir('/var/cpanel/users/') try: output.remove('system') except ValueError: pass self.users = output self.validate_users_to_move() # Creating a file for each user that wants to move. for single_user in self.users: open(self.base_dir + 'users/%s' % single_user, 'a').close() open(self.base_dir + 'files/%s.todo' % single_user, 'a').close() return output def validate_users_to_move(self): """ Void function. Its following rads.shared.is_cpanel_user in checking that /var/cpanel/users/username exists /var/cpanel/userdata/user exists That the user has a unix user on the server. :return: Updated the self.users on the system. """ temp_users = self.users userfile = '/var/cpanel/users/' userdata = '/var/cpanel/userdata/' for single_user in self.users: if not self.ssh_connection.file_exists(userfile + single_user): self.error("%s don't have userfile on the remote server." % single_user) temp_users.remove(single_user) pass if not self.ssh_connection.file_exists(userdata + single_user): self.error("%s doesn't have userdata folder." % single_user) temp_users.remove(single_user) pass if not self.ssh_connection.user_exists(single_user): self.error("%s doesn't have a unix user on the system." % single_user) temp_users.remove(single_user) pass self.users = temp_users def get_users(self): """ Getting users that we are trying to move from the users/ folder. :return: A list of each user we are moving. """ users_to_return = [] if os.path.exists(self.base_dir + 'users/'): for user in os.listdir(self.base_dir + 'users/'): users_to_return.append(user) return users_to_return def create_cpanel_backup(self): """ Creating backups of all the users. Doing skiphomedir by default for reliability. This ignores the system use which is a weird user in cPanel A file is created for each user that has had their backup created in backups/ :return: Names of all the backups created in an array. """ self.ssh_connection.check_connection() self.info("Entering Generate cPanel backups.") result = [] for single_user in self.get_users(): if 'system' not in single_user: self.info("Creating backup for %s." % single_user) output = self.ssh_connection.string_exec('/scripts/pkgacct --skiphomedir %s' % single_user) if 'pkgacctfile is' in output: backup_name = output.split("pkgacctfile is:")[1].split("\n")[0].strip() self.info("Created the backup %s" % backup_name) open(self.base_dir + 'backups/' + single_user + '.created', 'a').close() result.append(backup_name) else: self.warning("Backup wasn't created for %s." % single_user) return result def download_backups(self): """ Download all the backups on the remote server. Takes in backups of the class. Changes the name of all the backups downlaoded in backups/ to user.downloaded. :return: Downloads all the backups to /home/ """ self.ssh_connection.check_connection() backups = [] backups_to_download = [] self.info("[*] Entering Download backups.") # Getting the backups to download. if os.path.exists(self.base_dir + 'backups/'): backups_to_download = os.listdir(self.base_dir + 'backups/') # Getting users from backups created. for user in backups_to_download: if 'created' in user: user = user.split(".")[0] backups.append('cpmove-%s.tar.gz' % user) # Downloading the backups now. for files in backups: self.info("+++ Downloading %s" % files) self.ssh_connection.pull_files(remote_dir='/home/', local_dir='/home/', file_to_transfer=files) # checking for zero byte files. self.ssh_connection.ssh_exec('rm -f /home/%s' % files) # Renaming the files. for file in backups_to_download: user = file.split(".")[0] shutil.move(self.base_dir + 'backups/' + file, self.base_dir + 'backups/' + user + '.downloaded') return True def restore_cpanel_backup(self): """ The fun stuff. Basically going through and running /scripts/restorepkg on all the backups The users transfered should be listed in the users variable in the class, simply call that. Changes the name of each user in backups/ that had their backup restored to user.restored. From there a file is created in setup/ for that users files to be setup. :return: List of failed restores. """ self.info("[*] Entering Restore backups.") failed_users = {} users_to_restore = [] backups_to_restore = [] if os.path.exists(self.base_dir + 'backups/'): backups_to_restore = os.listdir(self.base_dir + 'backups/') for user in backups_to_restore: if 'downloaded' in user: user = user.split(".")[0] users_to_restore.append(user) for single_user in users_to_restore: if 'system' not in single_user: self.info("[*] Restoring %s" % single_user) command = ['/usr/local/cpanel/scripts/restorepkg', '--allow_reseller', '/home/cpmove-%s.tar.gz' % single_user] # Waiting seems to break things. The restore proc would Zombie itself while waiting # Not running restore_output.wait(). Instead doing .communicate() restore_output = subprocess.Popen(command, stdout=subprocess.PIPE) failed = False for line in restore_output.communicate()[0].split("\n"): if 'Account Restore Failed' in line: failed_users[single_user] = line failed = True elif 'already exists on this system' in line: failed_users[single_user] = line failed = True if not is_cpanel_user(single_user): # This checks that the user is actually there. # Checks /var/cpanel/user/single_user exists. # Checks /var/cpanel/userdata/single_user exists. # Lastly checks to see if there is a unix user. failed_users[single_user] += ": Check for user failed" failed = True if failed: self.warning("[x] !!! Failed to restore %s" % single_user) self.error(failed_users[str(single_user)]) else: self.info("[*] +++ Restored %s" % single_user) os.remove('/home/cpmove-%s.tar.gz' % single_user) shutil.move(self.base_dir + 'backups/%s.downloaded' % single_user, self.base_dir + 'backups/%s.restored' % single_user) open(self.base_dir + 'setup/%s.todo' % single_user, 'a').close() if not os.path.exists('/var/cpanel/users/%s' % single_user): if failed_users[single_user] is None: failed_users[single_user] = 'No /var/cpanel/users File' return failed_users def download_files(self): """ Function that downloads all of the users files. Rsync is being used due to the sftp function taking forever in paramiko. Downloads all the users files and changes the file name from user.todo to users.downloaded. :return: Downloads the files of the account. """ self.ssh_connection.check_connection() self.info("[*] Entering Download Files.") users_todo = [] users_to_download = [] if not os.path.exists('/home/xfer'): self.info(" Making /home/xfer") os.mkdir('/home/xfer') self.info("[*] Starting to copy /home/ to /home/xfer") if os.path.exists(self.base_dir + 'files/'): users_todo = os.listdir(self.base_dir + 'files/') for user in users_todo: if 'todo' in user: user = user.split(".")[0] users_to_download.append(user) for single_user in users_to_download: if not os.path.exists('/home/xfer/%s/' % single_user): os.mkdir('/home/xfer/%s/' % single_user) self.info("[*] Downloading %s's files" % single_user) self.ssh_connection.rsync_pull_files('/home/%s/' % single_user, '/home/xfer/%s' % single_user) shutil.move(self.base_dir + 'files/%s.todo' % single_user, self.base_dir + 'files/%s.downloaded' % single_user) return True def setup_users(self): """ Setting up the user. Moving /home/xfer/user to /home/user. After that, runs fixperms. Requires download_files to be called first. setup/user.todo is changes to setup/user.done. :return: ??? Not sure yet. """ self.info("[*] Entering Setup users") users_todo = [] users_to_setup = [] if os.path.exists(self.base_dir + 'setup/'): users_todo = os.listdir(self.base_dir + 'setup/') for user in users_todo: if 'todo' in user: user = user.split(".")[0] users_to_setup.append(user) for single_user in users_to_setup: self.info("[*] Setting up %s" % single_user) fixperms_command = ['/usr/bin/fixperms', single_user] # Holding old files to make sure nothing is deleted that is supposed to be deleted. shutil.move(os.path.join('/home/', single_user), '/home/%s.old' % single_user) shutil.move(os.path.join('/home/xfer/', single_user), os.path.join('/home/', single_user)) subprocess.Popen(fixperms_command).communicate() self.info("Setup %s" % single_user) shutil.move(self.base_dir + 'setup/%s.todo' % single_user, self.base_dir + 'setup/%s.done' % single_user) return True def transfer_ssls(self): """ Basically this will find all the ssl's in /home/$user/ssl and install them. The only way you can install SSL's effectively is via the WHM API. Calling it here. The hashs on the file names tell you what cert goes with what key. Per the API, the CAB is setup automatically. :return: A list of failed SSL installs. """ self.debug("Entering SSL install.") failed_installs = {} certs_to_install = {} keys_to_install = {} for user in self.get_users(): if os.path.exists('/home/%s/ssl/certs' % user): for cert in os.listdir('/home/%s/ssl/certs' % user): key_hash = cert.split('_')[-4:] # Grabbing domain from cert. # Each cert has 4 sections with hashed letters. # So from from the fourth section of the file from the back, we split the info. domain = '.'.join(cert.split('_')[:-4]) # Ran into an error where a customer had an SSL for a domain name that didn't exist. userdomains = open('/etc/userdomains') certs_to_install[domain] = open('/home/%s/ssl/certs/%s' % (user, cert)).read() for key in os.listdir('/home/%s/ssl/keys' % user): if key_hash[0] == key.split('_')[0]: keys_to_install[domain] = open('/home/%s/ssl/keys/%s' % (user, key)).read() if keys_to_install.get(domain) is None: self.warning("Key for %s not found." % domain) else: self.info("%s has no SSL's" % user) pass for cert in certs_to_install: # cert is the domain your installing the SSL for. self.info("Installing Cert for %s" % cert) # Ran into issues where the domain wasn't on the server and it attempted to install. userdomains = open('/etc/userdomains') if cert not in userdomains.readlines(): continue # API calls respond with "none" if the key is not updated. # Also checking that there is a cert and a key for each SSL. if certs_to_install[cert] and keys_to_install[cert]: api_call = whm_api('installssl', version=1, domain=cert, crt=certs_to_install[cert], key=keys_to_install[cert]) if api_success(api_call): self.info("Installed the SSL for %s" % cert) else: failed_installs[cert] = api_call self.critical(api_call) self.error("Failed to install SSL for %s" % cert) return failed_installs def check_urls(self): """ Checking the URL's of each of the users websites. Needs a list of users to be set first. :return: Return a dict of tuples containing the following: domain: (IP, HTML) """ self.debug("Entering Check Temp URL") result = {} checker = CheckUrl.CheckURL(ssh_connection=self.ssh_connection, logger=self.logger) checker.main() for domain in checker.result: result[domain] = (checker.domains[domain], checker.result[domain]) return result def reset(self): """ Reset's the VPS. Goes through users/ and removes any user in there. Removes /home/xfer. Removes /opt/cmu_ded. :return: Removes the mentioned files. """ self.debug("Entering cmumain.reset()") for single_user in self.get_users(): if 'system' not in single_user and is_cpanel_user(single_user): self.warning("Attempting to remove the cPanel user %s" % single_user) yes_exec = subprocess.Popen(['/bin/yes'], stdout=subprocess.PIPE, stderr=open(os.devnull)) removeacct_exec = subprocess.Popen(['/scripts/removeacct', single_user], stdin=yes_exec.stdout, stdout=subprocess.PIPE).communicate() # Closing stdout is important in avoiding SIGPIPE errors per documentation. yes_exec.stdout.close() self.warning("Removed %s" % single_user) self.warning("Removing /home/xfer") if os.path.exists('/home/xfer'): shutil.rmtree('/home/xfer') self.warning("Removing %s" % self.base_dir) if os.path.exists(self.base_dir): shutil.rmtree(self.base_dir)