From 9ca80141772591887845eeac68a9aff9f157d4c9 Mon Sep 17 00:00:00 2001 From: jingrow Date: Thu, 21 Aug 2025 18:08:53 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0jingrow=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 278 +- .pre-commit-config.yaml | 54 +- README.md | 82 +- agent/database_physical_restore.py | 1184 ++++----- agent/docker_cache_utils.py | 518 ++-- agent/templates/agent/nginx.conf.jinja2 | 602 ++--- agent/web.py | 3196 +++++++++++------------ setup.py | 44 +- 8 files changed, 2979 insertions(+), 2979 deletions(-) diff --git a/.gitignore b/.gitignore index 0bb0bfd..695436c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,140 +1,140 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. git.jingrow.com:3000/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# VSCode -.vscode/ - -# PyCharm -.idea/ - -# Vim -.vim/ - +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. git.jingrow.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode +.vscode/ + +# PyCharm +.idea/ + +# Vim +.vim/ + .DS_Store \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9615d1..0d0f9f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,27 @@ -exclude: 'node_modules|.git' -default_stages: [commit] -fail_fast: false - -repos: - - repo: http://git.jingrow.com:3000/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: debug-statements - - id: trailing-whitespace - files: 'agent.*' - exclude: '.*json$|.*txt$|.*csv|.*md|.*svg' - - id: check-merge-conflict - - id: check-ast - - id: check-json - - id: check-toml - - id: check-yaml - - - repo: http://git.jingrow.com:3000/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.6.5 - hooks: - # Run the linter. - - id: ruff - args: [--fix] - # Run the formatter. - - id: ruff-format +exclude: 'node_modules|.git' +default_stages: [commit] +fail_fast: false + +repos: + - repo: http://git.jingrow.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: debug-statements + - id: trailing-whitespace + files: 'agent.*' + exclude: '.*json$|.*txt$|.*csv|.*md|.*svg' + - id: check-merge-conflict + - id: check-ast + - id: check-json + - id: check-toml + - id: check-yaml + + - repo: http://git.jingrow.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.6.5 + hooks: + # Run the linter. + - id: ruff + args: [--fix] + # Run the formatter. + - id: ruff-format diff --git a/README.md b/README.md index 8323dbe..179d756 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,41 @@ -# Agent - -## Installation - -``` -mkdir agent && cd agent -git clone http://git.jingrow.com:3000/jingrow/agent repo -virtualenv env -source env/bin/activate -pip install -e ./repo -cp repo/redis.conf . -cp repo/Procfile . -``` - -## Running - -``` -honcho start -``` - -## CLI - -Agent has a CLI -([ref](http://git.jingrow.com:3000/jingrow/agent/blob/master/agent/cli.py)). You can -access this by activating the env: - -```bash -# Path to your agent's Python env might be different -source ./agent/env/bin/activate - -agent --help -``` - -Once you have activated the env, you can access the iPython console: - -```bash -agent console -``` - -This should have the server object instantiated if it was able to find the -`config.json` file. If not you can specify the path (check `agent console --help`). +# Agent + +## Installation + +``` +mkdir agent && cd agent +git clone http://git.jingrow.com/jingrow/agent repo +virtualenv env +source env/bin/activate +pip install -e ./repo +cp repo/redis.conf . +cp repo/Procfile . +``` + +## Running + +``` +honcho start +``` + +## CLI + +Agent has a CLI +([ref](http://git.jingrow.com/jingrow/agent/blob/master/agent/cli.py)). You can +access this by activating the env: + +```bash +# Path to your agent's Python env might be different +source ./agent/env/bin/activate + +agent --help +``` + +Once you have activated the env, you can access the iPython console: + +```bash +agent console +``` + +This should have the server object instantiated if it was able to find the +`config.json` file. If not you can specify the path (check `agent console --help`). diff --git a/agent/database_physical_restore.py b/agent/database_physical_restore.py index a75d873..141f744 100644 --- a/agent/database_physical_restore.py +++ b/agent/database_physical_restore.py @@ -1,592 +1,592 @@ -from __future__ import annotations - -import json -import os -import re -import shutil -import subprocess -from contextlib import suppress - -from agent.base import AgentException -from agent.database import CustomPeeweeDB -from agent.database_physical_backup import ( - DatabaseConnectionClosedWithDatabase, - get_path_of_physical_backup_metadata, - is_db_connection_usable, - kill_other_db_connections, - run_sql_query, -) -from agent.database_server import DatabaseServer -from agent.job import job, step -from agent.utils import compute_file_hash, get_mariadb_table_name_from_path - - -class DatabasePhysicalRestore(DatabaseServer): - def __init__( - self, - backup_db: str, - target_db: str, - target_db_root_password: str, - target_db_port: int, - target_db_host: str, - backup_db_base_directory: str, - target_db_base_directory: str = "/var/lib/mysql", - restore_specific_tables: bool = False, - tables_to_restore: list[str] | None = None, - ): - if tables_to_restore is None: - tables_to_restore = [] - - self._target_db_instance: CustomPeeweeDB = None - self._target_db_instance_connection_id: int = None - self._target_db_instance_for_myisam: CustomPeeweeDB = None - self._target_db_instance_for_myisam_connection_id: int = None - self.target_db = target_db - self.target_db_user = "root" - self.target_db_password = target_db_root_password - self.target_db_host = target_db_host - self.target_db_port = target_db_port - self.target_db_directory = os.path.join(target_db_base_directory, target_db) - - self.backup_db = backup_db - self.backup_db_base_directory = backup_db_base_directory - self.backup_db_directory = os.path.join(backup_db_base_directory, backup_db) - - self.restore_specific_tables = restore_specific_tables - self.tables_to_restore = tables_to_restore - - # Initialize the variables - self.files_metadata: dict[str, dict[str, str]] = {} - self.innodb_tables: list[str] = [] - self.myisam_tables: list[str] = [] - self.sequence_tables: list[str] = [] - - super().__init__() - - @job("Physical Restore Database") - def create_restore_job(self): - from filewarmer import FWUP - - self.fwup = FWUP() - self.validate_backup_files() - self.validate_connection_to_target_db() - self.warmup_myisam_files() - self.check_and_fix_myisam_table_files() - self.warmup_innodb_files() - # https://mariadb.com/kb/en/innodb-file-per-table-tablespaces/#importing-transportable-tablespaces-for-non-partitioned-tables - self.prepare_target_db_for_restore() - self.create_tables_from_table_schema() - self.discard_innodb_tablespaces_from_target_db() - self.perform_innodb_file_operations() - self.import_tablespaces_in_target_db() - self.hold_write_lock_on_myisam_tables() - self.perform_myisam_file_operations() - self.unlock_all_tables() - self._close_db_connections() - self.perform_post_restoration_validation_and_fixes() - - @step("Validate Backup Files") - def validate_backup_files(self): # noqa: C901 - # fetch the required metadata to proceed - backup_metadata_path = get_path_of_physical_backup_metadata( - self.backup_db_base_directory, self.backup_db - ) - if not os.path.exists(backup_metadata_path): - raise Exception(f"Backup metadata not found for {self.backup_db}") - - backup_metadata = None - with open(backup_metadata_path, "r") as f: - backup_metadata = json.load(f) - if not backup_metadata: - raise Exception(f"Backup metadata is empty for {self.backup_db}") - - self.files_metadata = backup_metadata["files_metadata"] - self.innodb_tables = backup_metadata["innodb_tables"] - self.myisam_tables = backup_metadata["myisam_tables"] - self.sequence_tables = backup_metadata.get("sequence_tables", []) - self.table_schema = backup_metadata["table_schema"] - if self.restore_specific_tables: - # remove invalid tables from tables_to_restore - all_tables = self.innodb_tables + self.myisam_tables - self.tables_to_restore = [table for table in self.tables_to_restore if table in all_tables] - - # remove the unwanted tables - self.innodb_tables = [table for table in self.innodb_tables if table in self.tables_to_restore] - self.myisam_tables = [table for table in self.myisam_tables if table in self.tables_to_restore] - - # validate files - files = os.listdir(self.backup_db_directory) - output = "" - invalid_files = set() - for file in files: - if not self.is_db_file_need_to_be_restored(file): - continue - if file not in self.files_metadata: - continue - file_metadata = self.files_metadata[file] - file_path = os.path.join(self.backup_db_directory, file) - # validate file size - file_size = os.path.getsize(file_path) - if file_size != file_metadata["size"]: - output += f"[INVALID] [FILE SIZE] {file} - {file_size} bytes\n" - invalid_files.add(file) - continue - - # if file checksum is provided, validate checksum - if file_metadata["checksum"]: - checksum = compute_file_hash(file_path, raise_exception=True) - if checksum != file_metadata["checksum"]: - output += f"[INVALID] [CHECKSUM] {file} - {checksum}\n" - invalid_files.add(file) - - if invalid_files: - output += "Invalid Files:\n" - for file in invalid_files: - output += f"{file}\n" - raise AgentException({"output": output}) - - return {"output": output} - - @step("Validate Connection to Target Database") - def validate_connection_to_target_db(self): - self._get_target_db().execute_sql("SELECT 1;") - self._get_target_db_for_myisam().execute_sql("SELECT 1;") - self._kill_other_active_db_connections() - - @step("Warmup MyISAM Files") - def warmup_myisam_files(self): - files = os.listdir(self.backup_db_directory) - files = [file for file in files if file.endswith(".MYI") or file.endswith(".MYD")] - file_paths = [os.path.join(self.backup_db_directory, file) for file in files] - file_paths = [file for file in file_paths if self.is_db_file_need_to_be_restored(file)] - self._warmup_files(file_paths) - - @step("Check and Fix MyISAM Table Files") - def check_and_fix_myisam_table_files(self): - """ - Check issues in MyISAM table files - myisamchk :path - - If any issues found, try to repair the table - """ - files = os.listdir(self.backup_db_directory) - files = [file for file in files if file.endswith(".MYI")] - files = [file for file in files if self.is_db_file_need_to_be_restored(file)] - for file in files: - myisamchk_command = [ - "myisamchk", - os.path.join(self.backup_db_directory, file), - ] - try: - subprocess.check_output(myisamchk_command) - except subprocess.CalledProcessError: - myisamchk_command.append("--recover") - try: - subprocess.check_output(myisamchk_command) - except subprocess.CalledProcessError as e: - print(f"Error while repairing MyISAM table file: {e.output}") - print("Stopping the process") - raise Exception from e - - self._get_target_db_for_myisam().execute_sql("UNLOCK TABLES;") - - @step("Warmup InnoDB Files") - def warmup_innodb_files(self): - files = os.listdir(self.backup_db_directory) - files = [file for file in files if file.endswith(".ibd")] - file_paths = [os.path.join(self.backup_db_directory, file) for file in files] - file_paths = [file for file in file_paths if self.is_db_file_need_to_be_restored(file)] - self._warmup_files(file_paths) - - @step("Prepare Database for Restoration") - def prepare_target_db_for_restore(self): - # Only perform this, if we are restoring all tables - if self.restore_specific_tables: - return - - """ - Prepare the database for import - - fetch existing tables list in database - - delete all tables - """ - tables = self._get_target_db().get_tables() - # before start dropping tables, disable foreign key checks - # it will reduce the time to drop tables and will not cause any block while dropping tables - self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 0;") - for table in tables: - self._kill_other_active_db_connections() - self._get_target_db().execute_sql(self.get_drop_table_statement(table)) - self._get_target_db().execute_sql( - "SET SESSION FOREIGN_KEY_CHECKS = 1;" - ) # re-enable foreign key checks - - @step("Create Tables from Table Schema") - def create_tables_from_table_schema(self): - if self.restore_specific_tables: - sql_stmts = [] - for table in self.tables_to_restore: - sql_stmts.append(self.get_drop_table_statement(table)) - sql_stmts.append(self.get_create_table_statement(self.table_schema, table)) - else: - # http://git.jingrow.com:3000/jingrow/jingrow/pull/26855 - schema_file_content: str = re.sub( - r"/\*M{0,1}!999999\\- enable the sandbox mode \*/", - "", - self.table_schema, - ) - # # http://git.jingrow.com:3000/jingrow/jingrow/pull/28879 - schema_file_content: str = re.sub( - r"/\*![0-9]* DEFINER=[^ ]* SQL SECURITY DEFINER \*/", - "", - self.table_schema, - ) - # create the tables - sql_stmts = schema_file_content.split(";\n") - - # before start dropping tables, disable foreign key checks - # it will reduce the time to drop tables and will not cause any block while dropping tables - self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 0;") - - # kill other active db connections - self._kill_other_active_db_connections() - - # Drop and re-create the tables - for sql_stmt in sql_stmts: - if sql_stmt.strip(): - self._get_target_db().execute_sql(sql_stmt) - - # re-enable foreign key checks - self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 1;") - - @step("Discard InnoDB Tablespaces") - def discard_innodb_tablespaces_from_target_db(self): - # https://mariadb.com/kb/en/innodb-file-per-table-tablespaces/#foreign-key-constraints - self._get_target_db().execute_sql("SET SESSION foreign_key_checks = 0;") - for table in self.innodb_tables: - self._kill_other_active_db_connections() - self._get_target_db().execute_sql(f"ALTER TABLE `{table}` DISCARD TABLESPACE;") - self._get_target_db().execute_sql( - "SET SESSION foreign_key_checks = 1;" - ) # re-enable foreign key checks - - @step("Copying InnoDB Table Files") - def perform_innodb_file_operations(self): - self._perform_file_operations(engine="innodb") - - @step("Import InnoDB Tablespaces") - def import_tablespaces_in_target_db(self): - for table in self.innodb_tables: - self._kill_other_active_db_connections() - self._get_target_db().execute_sql(f"ALTER TABLE `{table}` IMPORT TABLESPACE;") - - @step("Hold Write Lock on MyISAM Tables") - def hold_write_lock_on_myisam_tables(self): - """ - MyISAM doesn't support foreign key constraints - So, need to take write lock on MyISAM tables - - Discard tablespace query on innodb already took care of locks - """ - if not self.myisam_tables: - return - tables = [f"`{table}` WRITE" for table in self.myisam_tables] - self._kill_other_active_db_connections() - self._get_target_db_for_myisam().execute_sql("LOCK TABLES {};".format(", ".join(tables))) - - @step("Copying MyISAM Table Files") - def perform_myisam_file_operations(self): - self._perform_file_operations(engine="myisam") - - @step("Unlock All Tables") - def unlock_all_tables(self): - self._get_target_db().execute_sql("UNLOCK TABLES;") - self._get_target_db_for_myisam().execute_sql("UNLOCK TABLES;") - - @step("Validate And Fix Tables") - def perform_post_restoration_validation_and_fixes(self): - innodb_tables_with_fts = self._get_innodb_tables_with_fts_index() - """ - FLUSH TABLES ... FOR EXPORT does not support FULLTEXT indexes. - https://dev.mysql.com/doc/refman/8.4/en/innodb-table-import.html#:~:text=in%20the%20operation.-,Limitations,-The%20Transportable%20Tablespaces - - Need to drop all fulltext indexes of InnoDB tables. - Then, optimize table to fix existing corruptions and rebuild table (if needed). - Then, recreate the fulltext indexes. - """ - - for table in innodb_tables_with_fts: - """ - No need to waste time on checking whether index is corrupted or not - Because, physical restoration will not work for FULLTEXT index. - """ - self.recreate_fts_indexes(table) - - """ - MyISAM table corruption can generally happen due to mismatch of no of records in MYD file. - - myisamchk can't find and fix this issue. - Because this out of sync happen after creating a blank MyISAM table and just copying MYF & MYI files. - - Usually, DB Restart will fix this issue. But we can't do in live database. - So running `REPAIR TABLE ... USE_FRM` can fix the issue. - https://dev.mysql.com/doc/refman/8.4/en/myisam-repair.html - """ - for table in self.myisam_tables: - if self.is_table_corrupted(table) and not self.repair_myisam_table(table): - raise Exception(f"Failed to repair table {table}") - - def _warmup_files(self, file_paths: list[str]): - """ - Once the snapshot is converted to disk and attached to the instance, - All the files are not immediately available to the system. - - AWS EBS volumes are lazily loaded from S3. - - So, before doing any operations on the disk, need to warm up the disk. - But, we will selectively warm up only required blocks. - - Ref - https://docs.aws.amazon.com/ebs/latest/userguide/ebs-initialize.html - """ - - self.fwup.warmup(file_paths, method="io_uring") - - def _perform_file_operations(self, engine: str): - for file in os.listdir(self.backup_db_directory): - # skip if file is not need to be restored - if not self.is_db_file_need_to_be_restored(file): - continue - - # copy only .ibd, .cfg if innodb - if engine == "innodb" and not (file.endswith(".ibd") or file.endswith(".cfg")): - continue - - # copy one .MYI, .MYD files if myisam - if engine == "myisam" and not (file.endswith(".MYI") or file.endswith(".MYD")): - continue - - shutil.copyfile( - os.path.join(self.backup_db_directory, file), - os.path.join(self.target_db_directory, file), - ) - - def is_table_need_to_be_restored(self, table_name: str) -> bool: - if not self.restore_specific_tables: - # Ensure the tale_name is not a FTS table - return not re.search( - r"(FTS|fts)_[0-9a-f]+_(?:[0-9a-f]+_)?(?:INDEX|index|BEING|being|CONFIG|config|DELETED|deleted)", - table_name, - ) - return table_name in self.innodb_tables or table_name in self.myisam_tables - - def is_db_file_need_to_be_restored(self, file_name: str) -> bool: - return self.is_table_need_to_be_restored(get_mariadb_table_name_from_path(file_name)) - - def is_sequence_table(self, table_name: str) -> bool: - return table_name in self.sequence_tables - - def get_create_table_statement(self, sql_dump, table_name) -> str: - if self.is_sequence_table(table_name): - # Define the regex pattern to match the CREATE SEQUENCE statement - pattern = re.compile(rf"CREATE SEQUENCE `{table_name}`[\s\S]*?;", re.DOTALL) - else: - # Define the regex pattern to match the CREATE TABLE statement - pattern = re.compile(rf"CREATE TABLE `{table_name}`[\s\S]*?;(?=\s*(?=\n|$))", re.DOTALL) - - # Search for the CREATE TABLE statement in the SQL dump - match = pattern.search(sql_dump) - if match: - return match.group(0) - - if self.is_sequence_table(table_name): - raise Exception(f"CREATE SEQUENCE statement for {table_name} not found in SQL dump") - else: # noqa: RET506 - raise Exception(f"CREATE TABLE statement for {table_name} not found in SQL dump") - - def get_drop_table_statement(self, table_name) -> str: - if self.is_sequence_table(table_name): - return f"DROP SEQUENCE IF EXISTS `{table_name}`;" - - return f"DROP TABLE IF EXISTS `{table_name}`;" - - def is_table_corrupted(self, table_name: str) -> bool: - result = run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f"CHECK TABLE `{table_name}` QUICK;", - ) - """ - +-----------------------------------+-------+----------+------------------------------------------------------+ - | Table | Op | Msg_type | Msg_text | - +-----------------------------------+-------+----------+------------------------------------------------------+ - | _8edd549f4b072174.__global_search | check | warning | Size of indexfile is: 22218752 Should be: 4096 | - | _8edd549f4b072174.__global_search | check | warning | Size of datafile is: 31303496 Should be: 0 | - | _8edd549f4b072174.__global_search | check | error | Record-count is not ok; is 152774 Should be: 0 | - | _8edd549f4b072174.__global_search | check | warning | Found 172605 key parts. Should be: 0 | - | _8edd549f4b072174.__global_search | check | error | Corrupt | - +-----------------------------------+-------+----------+------------------------------------------------------+ - - +-------------------------------------------+-------+----------+--------------------------------------------------------+ - | Table | Op | Msg_type | Msg_text | - +-------------------------------------------+-------+----------+--------------------------------------------------------+ - | _8edd549f4b072174.energy_point_log_id_seq | check | note | The storage engine for the table doesn't support check | - +-------------------------------------------+-------+----------+--------------------------------------------------------+ - """ # noqa: E501 - isError = False - for row in result: - if row[2] == "error" or (row[2] == "warning" and table_name in self.myisam_tables): - isError = True - break - return isError - - def repair_myisam_table(self, table_name: str) -> bool: - self._kill_other_active_db_connections() - result = run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f"REPAIR TABLE `{table_name}` USE_FRM;", - ) - """ - +---------------------------------------------------+--------+----------+----------+ - | Table | Op | Msg_type | Msg_text | - +---------------------------------------------------+--------+----------+----------+ - | _8edd549f4b072174.tabInsights Query Execution Log | repair | status | OK | - +---------------------------------------------------+--------+----------+----------+ - - Msg Type can be status, error, info, note, or warning - """ - isErrorOccurred = False - for row in result: - if row[2] == "error": - isErrorOccurred = True - break - - return not isErrorOccurred - - def recreate_fts_indexes(self, table: str): - fts_indexes = self._get_fts_indexes_of_table(table) - for index_name, _ in fts_indexes.items(): - self._kill_other_active_db_connections() - run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f"ALTER TABLE `{table}` DROP INDEX IF EXISTS `{index_name}`;", - ) - # Optimize table to fix existing corruptions - self._kill_other_active_db_connections() - run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), f"OPTIMIZE TABLE `{table}`;" - ) - # Recreate the indexes - for index_name, columns in fts_indexes.items(): - self._kill_other_active_db_connections() - run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f"ALTER TABLE `{table}` ADD FULLTEXT INDEX `{index_name}` ({columns});", - ) - - def _get_innodb_tables_with_fts_index(self): - rows = run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f""" - SELECT - DISTINCT(t.TABLE_NAME) - FROM - information_schema.STATISTICS s - JOIN - information_schema.TABLES t - ON s.TABLE_SCHEMA = t.TABLE_SCHEMA - AND s.TABLE_NAME = t.TABLE_NAME - WHERE - s.INDEX_TYPE = 'FULLTEXT' - AND t.TABLE_SCHEMA = '{self.target_db}' - AND t.ENGINE = 'InnoDB'; - """, - ) - return [row[0] for row in rows] - - def _get_fts_indexes_of_table(self, table: str) -> dict[str, str]: - rows = run_sql_query( - self._get_target_db(raise_error_on_connection_closed=False), - f""" - SELECT - INDEX_NAME, group_concat(column_name ORDER BY seq_in_index) AS columns - FROM - information_schema.statistics - WHERE - TABLE_SCHEMA = '{self.target_db}' - AND TABLE_NAME = '{table}' - AND INDEX_TYPE = 'FULLTEXT' - GROUP BY - INDEX_NAME; - """, - ) - return {row[0]: row[1] for row in rows} - - def _get_target_db(self, raise_error_on_connection_closed: bool = True) -> CustomPeeweeDB: - if self._target_db_instance is not None and not is_db_connection_usable(self._target_db_instance): - if raise_error_on_connection_closed: - raise DatabaseConnectionClosedWithDatabase() - self._target_db_instance = None - self._target_db_instance_connection_id = None - - if self._target_db_instance is not None: - return self._target_db_instance - - self._target_db_instance = CustomPeeweeDB( - self.target_db, - user=self.target_db_user, - password=self.target_db_password, - host=self.target_db_host, - port=self.target_db_port, - ) - self._target_db_instance.connect() - # Set session wait timeout to 4 hours [EXPERIMENTAL] - self._target_db_instance.execute_sql("SET SESSION wait_timeout = 14400;") - # Fetch the connection id - self._target_db_instance_connection_id = int( - self._target_db_instance.execute_sql("SELECT CONNECTION_ID();").fetchone()[0] - ) - return self._target_db_instance - - def _get_target_db_for_myisam(self) -> CustomPeeweeDB: - if self._target_db_instance_for_myisam is not None: - if not is_db_connection_usable(self._target_db_instance_for_myisam): - raise DatabaseConnectionClosedWithDatabase() - return self._target_db_instance_for_myisam - - self._target_db_instance_for_myisam = CustomPeeweeDB( - self.target_db, - user=self.target_db_user, - password=self.target_db_password, - host=self.target_db_host, - port=self.target_db_port, - autocommit=False, - ) - self._target_db_instance_for_myisam.connect() - # Set session wait timeout to 4 hours [EXPERIMENTAL] - self._target_db_instance_for_myisam.execute_sql("SET SESSION wait_timeout = 14400;") - # Fetch the connection id - self._target_db_instance_for_myisam_connection_id = int( - self._target_db_instance_for_myisam.execute_sql("SELECT CONNECTION_ID();").fetchone()[0] - ) - return self._target_db_instance_for_myisam - - def _close_db_connections(self): - if self._target_db_instance is not None: - with suppress(Exception): - self._target_db_instance.close() - if self._target_db_instance_for_myisam is not None: - with suppress(Exception): - self._target_db_instance_for_myisam.close() - - def _kill_other_active_db_connections(self): - current_connection_ids = [] - if self._target_db_instance is not None: - current_connection_ids.append(self._target_db_instance_connection_id) - if self._target_db_instance_for_myisam is not None: - current_connection_ids.append(self._target_db_instance_for_myisam_connection_id) - - if not current_connection_ids: - return - - kill_other_db_connections(self._target_db_instance, current_connection_ids) - - def __del__(self): - self._close_db_connections() +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from contextlib import suppress + +from agent.base import AgentException +from agent.database import CustomPeeweeDB +from agent.database_physical_backup import ( + DatabaseConnectionClosedWithDatabase, + get_path_of_physical_backup_metadata, + is_db_connection_usable, + kill_other_db_connections, + run_sql_query, +) +from agent.database_server import DatabaseServer +from agent.job import job, step +from agent.utils import compute_file_hash, get_mariadb_table_name_from_path + + +class DatabasePhysicalRestore(DatabaseServer): + def __init__( + self, + backup_db: str, + target_db: str, + target_db_root_password: str, + target_db_port: int, + target_db_host: str, + backup_db_base_directory: str, + target_db_base_directory: str = "/var/lib/mysql", + restore_specific_tables: bool = False, + tables_to_restore: list[str] | None = None, + ): + if tables_to_restore is None: + tables_to_restore = [] + + self._target_db_instance: CustomPeeweeDB = None + self._target_db_instance_connection_id: int = None + self._target_db_instance_for_myisam: CustomPeeweeDB = None + self._target_db_instance_for_myisam_connection_id: int = None + self.target_db = target_db + self.target_db_user = "root" + self.target_db_password = target_db_root_password + self.target_db_host = target_db_host + self.target_db_port = target_db_port + self.target_db_directory = os.path.join(target_db_base_directory, target_db) + + self.backup_db = backup_db + self.backup_db_base_directory = backup_db_base_directory + self.backup_db_directory = os.path.join(backup_db_base_directory, backup_db) + + self.restore_specific_tables = restore_specific_tables + self.tables_to_restore = tables_to_restore + + # Initialize the variables + self.files_metadata: dict[str, dict[str, str]] = {} + self.innodb_tables: list[str] = [] + self.myisam_tables: list[str] = [] + self.sequence_tables: list[str] = [] + + super().__init__() + + @job("Physical Restore Database") + def create_restore_job(self): + from filewarmer import FWUP + + self.fwup = FWUP() + self.validate_backup_files() + self.validate_connection_to_target_db() + self.warmup_myisam_files() + self.check_and_fix_myisam_table_files() + self.warmup_innodb_files() + # https://mariadb.com/kb/en/innodb-file-per-table-tablespaces/#importing-transportable-tablespaces-for-non-partitioned-tables + self.prepare_target_db_for_restore() + self.create_tables_from_table_schema() + self.discard_innodb_tablespaces_from_target_db() + self.perform_innodb_file_operations() + self.import_tablespaces_in_target_db() + self.hold_write_lock_on_myisam_tables() + self.perform_myisam_file_operations() + self.unlock_all_tables() + self._close_db_connections() + self.perform_post_restoration_validation_and_fixes() + + @step("Validate Backup Files") + def validate_backup_files(self): # noqa: C901 + # fetch the required metadata to proceed + backup_metadata_path = get_path_of_physical_backup_metadata( + self.backup_db_base_directory, self.backup_db + ) + if not os.path.exists(backup_metadata_path): + raise Exception(f"Backup metadata not found for {self.backup_db}") + + backup_metadata = None + with open(backup_metadata_path, "r") as f: + backup_metadata = json.load(f) + if not backup_metadata: + raise Exception(f"Backup metadata is empty for {self.backup_db}") + + self.files_metadata = backup_metadata["files_metadata"] + self.innodb_tables = backup_metadata["innodb_tables"] + self.myisam_tables = backup_metadata["myisam_tables"] + self.sequence_tables = backup_metadata.get("sequence_tables", []) + self.table_schema = backup_metadata["table_schema"] + if self.restore_specific_tables: + # remove invalid tables from tables_to_restore + all_tables = self.innodb_tables + self.myisam_tables + self.tables_to_restore = [table for table in self.tables_to_restore if table in all_tables] + + # remove the unwanted tables + self.innodb_tables = [table for table in self.innodb_tables if table in self.tables_to_restore] + self.myisam_tables = [table for table in self.myisam_tables if table in self.tables_to_restore] + + # validate files + files = os.listdir(self.backup_db_directory) + output = "" + invalid_files = set() + for file in files: + if not self.is_db_file_need_to_be_restored(file): + continue + if file not in self.files_metadata: + continue + file_metadata = self.files_metadata[file] + file_path = os.path.join(self.backup_db_directory, file) + # validate file size + file_size = os.path.getsize(file_path) + if file_size != file_metadata["size"]: + output += f"[INVALID] [FILE SIZE] {file} - {file_size} bytes\n" + invalid_files.add(file) + continue + + # if file checksum is provided, validate checksum + if file_metadata["checksum"]: + checksum = compute_file_hash(file_path, raise_exception=True) + if checksum != file_metadata["checksum"]: + output += f"[INVALID] [CHECKSUM] {file} - {checksum}\n" + invalid_files.add(file) + + if invalid_files: + output += "Invalid Files:\n" + for file in invalid_files: + output += f"{file}\n" + raise AgentException({"output": output}) + + return {"output": output} + + @step("Validate Connection to Target Database") + def validate_connection_to_target_db(self): + self._get_target_db().execute_sql("SELECT 1;") + self._get_target_db_for_myisam().execute_sql("SELECT 1;") + self._kill_other_active_db_connections() + + @step("Warmup MyISAM Files") + def warmup_myisam_files(self): + files = os.listdir(self.backup_db_directory) + files = [file for file in files if file.endswith(".MYI") or file.endswith(".MYD")] + file_paths = [os.path.join(self.backup_db_directory, file) for file in files] + file_paths = [file for file in file_paths if self.is_db_file_need_to_be_restored(file)] + self._warmup_files(file_paths) + + @step("Check and Fix MyISAM Table Files") + def check_and_fix_myisam_table_files(self): + """ + Check issues in MyISAM table files + myisamchk :path + + If any issues found, try to repair the table + """ + files = os.listdir(self.backup_db_directory) + files = [file for file in files if file.endswith(".MYI")] + files = [file for file in files if self.is_db_file_need_to_be_restored(file)] + for file in files: + myisamchk_command = [ + "myisamchk", + os.path.join(self.backup_db_directory, file), + ] + try: + subprocess.check_output(myisamchk_command) + except subprocess.CalledProcessError: + myisamchk_command.append("--recover") + try: + subprocess.check_output(myisamchk_command) + except subprocess.CalledProcessError as e: + print(f"Error while repairing MyISAM table file: {e.output}") + print("Stopping the process") + raise Exception from e + + self._get_target_db_for_myisam().execute_sql("UNLOCK TABLES;") + + @step("Warmup InnoDB Files") + def warmup_innodb_files(self): + files = os.listdir(self.backup_db_directory) + files = [file for file in files if file.endswith(".ibd")] + file_paths = [os.path.join(self.backup_db_directory, file) for file in files] + file_paths = [file for file in file_paths if self.is_db_file_need_to_be_restored(file)] + self._warmup_files(file_paths) + + @step("Prepare Database for Restoration") + def prepare_target_db_for_restore(self): + # Only perform this, if we are restoring all tables + if self.restore_specific_tables: + return + + """ + Prepare the database for import + - fetch existing tables list in database + - delete all tables + """ + tables = self._get_target_db().get_tables() + # before start dropping tables, disable foreign key checks + # it will reduce the time to drop tables and will not cause any block while dropping tables + self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 0;") + for table in tables: + self._kill_other_active_db_connections() + self._get_target_db().execute_sql(self.get_drop_table_statement(table)) + self._get_target_db().execute_sql( + "SET SESSION FOREIGN_KEY_CHECKS = 1;" + ) # re-enable foreign key checks + + @step("Create Tables from Table Schema") + def create_tables_from_table_schema(self): + if self.restore_specific_tables: + sql_stmts = [] + for table in self.tables_to_restore: + sql_stmts.append(self.get_drop_table_statement(table)) + sql_stmts.append(self.get_create_table_statement(self.table_schema, table)) + else: + # http://git.jingrow.com/jingrow/jingrow/pull/26855 + schema_file_content: str = re.sub( + r"/\*M{0,1}!999999\\- enable the sandbox mode \*/", + "", + self.table_schema, + ) + # # http://git.jingrow.com/jingrow/jingrow/pull/28879 + schema_file_content: str = re.sub( + r"/\*![0-9]* DEFINER=[^ ]* SQL SECURITY DEFINER \*/", + "", + self.table_schema, + ) + # create the tables + sql_stmts = schema_file_content.split(";\n") + + # before start dropping tables, disable foreign key checks + # it will reduce the time to drop tables and will not cause any block while dropping tables + self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 0;") + + # kill other active db connections + self._kill_other_active_db_connections() + + # Drop and re-create the tables + for sql_stmt in sql_stmts: + if sql_stmt.strip(): + self._get_target_db().execute_sql(sql_stmt) + + # re-enable foreign key checks + self._get_target_db().execute_sql("SET SESSION FOREIGN_KEY_CHECKS = 1;") + + @step("Discard InnoDB Tablespaces") + def discard_innodb_tablespaces_from_target_db(self): + # https://mariadb.com/kb/en/innodb-file-per-table-tablespaces/#foreign-key-constraints + self._get_target_db().execute_sql("SET SESSION foreign_key_checks = 0;") + for table in self.innodb_tables: + self._kill_other_active_db_connections() + self._get_target_db().execute_sql(f"ALTER TABLE `{table}` DISCARD TABLESPACE;") + self._get_target_db().execute_sql( + "SET SESSION foreign_key_checks = 1;" + ) # re-enable foreign key checks + + @step("Copying InnoDB Table Files") + def perform_innodb_file_operations(self): + self._perform_file_operations(engine="innodb") + + @step("Import InnoDB Tablespaces") + def import_tablespaces_in_target_db(self): + for table in self.innodb_tables: + self._kill_other_active_db_connections() + self._get_target_db().execute_sql(f"ALTER TABLE `{table}` IMPORT TABLESPACE;") + + @step("Hold Write Lock on MyISAM Tables") + def hold_write_lock_on_myisam_tables(self): + """ + MyISAM doesn't support foreign key constraints + So, need to take write lock on MyISAM tables + + Discard tablespace query on innodb already took care of locks + """ + if not self.myisam_tables: + return + tables = [f"`{table}` WRITE" for table in self.myisam_tables] + self._kill_other_active_db_connections() + self._get_target_db_for_myisam().execute_sql("LOCK TABLES {};".format(", ".join(tables))) + + @step("Copying MyISAM Table Files") + def perform_myisam_file_operations(self): + self._perform_file_operations(engine="myisam") + + @step("Unlock All Tables") + def unlock_all_tables(self): + self._get_target_db().execute_sql("UNLOCK TABLES;") + self._get_target_db_for_myisam().execute_sql("UNLOCK TABLES;") + + @step("Validate And Fix Tables") + def perform_post_restoration_validation_and_fixes(self): + innodb_tables_with_fts = self._get_innodb_tables_with_fts_index() + """ + FLUSH TABLES ... FOR EXPORT does not support FULLTEXT indexes. + https://dev.mysql.com/doc/refman/8.4/en/innodb-table-import.html#:~:text=in%20the%20operation.-,Limitations,-The%20Transportable%20Tablespaces + + Need to drop all fulltext indexes of InnoDB tables. + Then, optimize table to fix existing corruptions and rebuild table (if needed). + Then, recreate the fulltext indexes. + """ + + for table in innodb_tables_with_fts: + """ + No need to waste time on checking whether index is corrupted or not + Because, physical restoration will not work for FULLTEXT index. + """ + self.recreate_fts_indexes(table) + + """ + MyISAM table corruption can generally happen due to mismatch of no of records in MYD file. + + myisamchk can't find and fix this issue. + Because this out of sync happen after creating a blank MyISAM table and just copying MYF & MYI files. + + Usually, DB Restart will fix this issue. But we can't do in live database. + So running `REPAIR TABLE ... USE_FRM` can fix the issue. + https://dev.mysql.com/doc/refman/8.4/en/myisam-repair.html + """ + for table in self.myisam_tables: + if self.is_table_corrupted(table) and not self.repair_myisam_table(table): + raise Exception(f"Failed to repair table {table}") + + def _warmup_files(self, file_paths: list[str]): + """ + Once the snapshot is converted to disk and attached to the instance, + All the files are not immediately available to the system. + + AWS EBS volumes are lazily loaded from S3. + + So, before doing any operations on the disk, need to warm up the disk. + But, we will selectively warm up only required blocks. + + Ref - https://docs.aws.amazon.com/ebs/latest/userguide/ebs-initialize.html + """ + + self.fwup.warmup(file_paths, method="io_uring") + + def _perform_file_operations(self, engine: str): + for file in os.listdir(self.backup_db_directory): + # skip if file is not need to be restored + if not self.is_db_file_need_to_be_restored(file): + continue + + # copy only .ibd, .cfg if innodb + if engine == "innodb" and not (file.endswith(".ibd") or file.endswith(".cfg")): + continue + + # copy one .MYI, .MYD files if myisam + if engine == "myisam" and not (file.endswith(".MYI") or file.endswith(".MYD")): + continue + + shutil.copyfile( + os.path.join(self.backup_db_directory, file), + os.path.join(self.target_db_directory, file), + ) + + def is_table_need_to_be_restored(self, table_name: str) -> bool: + if not self.restore_specific_tables: + # Ensure the tale_name is not a FTS table + return not re.search( + r"(FTS|fts)_[0-9a-f]+_(?:[0-9a-f]+_)?(?:INDEX|index|BEING|being|CONFIG|config|DELETED|deleted)", + table_name, + ) + return table_name in self.innodb_tables or table_name in self.myisam_tables + + def is_db_file_need_to_be_restored(self, file_name: str) -> bool: + return self.is_table_need_to_be_restored(get_mariadb_table_name_from_path(file_name)) + + def is_sequence_table(self, table_name: str) -> bool: + return table_name in self.sequence_tables + + def get_create_table_statement(self, sql_dump, table_name) -> str: + if self.is_sequence_table(table_name): + # Define the regex pattern to match the CREATE SEQUENCE statement + pattern = re.compile(rf"CREATE SEQUENCE `{table_name}`[\s\S]*?;", re.DOTALL) + else: + # Define the regex pattern to match the CREATE TABLE statement + pattern = re.compile(rf"CREATE TABLE `{table_name}`[\s\S]*?;(?=\s*(?=\n|$))", re.DOTALL) + + # Search for the CREATE TABLE statement in the SQL dump + match = pattern.search(sql_dump) + if match: + return match.group(0) + + if self.is_sequence_table(table_name): + raise Exception(f"CREATE SEQUENCE statement for {table_name} not found in SQL dump") + else: # noqa: RET506 + raise Exception(f"CREATE TABLE statement for {table_name} not found in SQL dump") + + def get_drop_table_statement(self, table_name) -> str: + if self.is_sequence_table(table_name): + return f"DROP SEQUENCE IF EXISTS `{table_name}`;" + + return f"DROP TABLE IF EXISTS `{table_name}`;" + + def is_table_corrupted(self, table_name: str) -> bool: + result = run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f"CHECK TABLE `{table_name}` QUICK;", + ) + """ + +-----------------------------------+-------+----------+------------------------------------------------------+ + | Table | Op | Msg_type | Msg_text | + +-----------------------------------+-------+----------+------------------------------------------------------+ + | _8edd549f4b072174.__global_search | check | warning | Size of indexfile is: 22218752 Should be: 4096 | + | _8edd549f4b072174.__global_search | check | warning | Size of datafile is: 31303496 Should be: 0 | + | _8edd549f4b072174.__global_search | check | error | Record-count is not ok; is 152774 Should be: 0 | + | _8edd549f4b072174.__global_search | check | warning | Found 172605 key parts. Should be: 0 | + | _8edd549f4b072174.__global_search | check | error | Corrupt | + +-----------------------------------+-------+----------+------------------------------------------------------+ + + +-------------------------------------------+-------+----------+--------------------------------------------------------+ + | Table | Op | Msg_type | Msg_text | + +-------------------------------------------+-------+----------+--------------------------------------------------------+ + | _8edd549f4b072174.energy_point_log_id_seq | check | note | The storage engine for the table doesn't support check | + +-------------------------------------------+-------+----------+--------------------------------------------------------+ + """ # noqa: E501 + isError = False + for row in result: + if row[2] == "error" or (row[2] == "warning" and table_name in self.myisam_tables): + isError = True + break + return isError + + def repair_myisam_table(self, table_name: str) -> bool: + self._kill_other_active_db_connections() + result = run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f"REPAIR TABLE `{table_name}` USE_FRM;", + ) + """ + +---------------------------------------------------+--------+----------+----------+ + | Table | Op | Msg_type | Msg_text | + +---------------------------------------------------+--------+----------+----------+ + | _8edd549f4b072174.tabInsights Query Execution Log | repair | status | OK | + +---------------------------------------------------+--------+----------+----------+ + + Msg Type can be status, error, info, note, or warning + """ + isErrorOccurred = False + for row in result: + if row[2] == "error": + isErrorOccurred = True + break + + return not isErrorOccurred + + def recreate_fts_indexes(self, table: str): + fts_indexes = self._get_fts_indexes_of_table(table) + for index_name, _ in fts_indexes.items(): + self._kill_other_active_db_connections() + run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f"ALTER TABLE `{table}` DROP INDEX IF EXISTS `{index_name}`;", + ) + # Optimize table to fix existing corruptions + self._kill_other_active_db_connections() + run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), f"OPTIMIZE TABLE `{table}`;" + ) + # Recreate the indexes + for index_name, columns in fts_indexes.items(): + self._kill_other_active_db_connections() + run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f"ALTER TABLE `{table}` ADD FULLTEXT INDEX `{index_name}` ({columns});", + ) + + def _get_innodb_tables_with_fts_index(self): + rows = run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f""" + SELECT + DISTINCT(t.TABLE_NAME) + FROM + information_schema.STATISTICS s + JOIN + information_schema.TABLES t + ON s.TABLE_SCHEMA = t.TABLE_SCHEMA + AND s.TABLE_NAME = t.TABLE_NAME + WHERE + s.INDEX_TYPE = 'FULLTEXT' + AND t.TABLE_SCHEMA = '{self.target_db}' + AND t.ENGINE = 'InnoDB'; + """, + ) + return [row[0] for row in rows] + + def _get_fts_indexes_of_table(self, table: str) -> dict[str, str]: + rows = run_sql_query( + self._get_target_db(raise_error_on_connection_closed=False), + f""" + SELECT + INDEX_NAME, group_concat(column_name ORDER BY seq_in_index) AS columns + FROM + information_schema.statistics + WHERE + TABLE_SCHEMA = '{self.target_db}' + AND TABLE_NAME = '{table}' + AND INDEX_TYPE = 'FULLTEXT' + GROUP BY + INDEX_NAME; + """, + ) + return {row[0]: row[1] for row in rows} + + def _get_target_db(self, raise_error_on_connection_closed: bool = True) -> CustomPeeweeDB: + if self._target_db_instance is not None and not is_db_connection_usable(self._target_db_instance): + if raise_error_on_connection_closed: + raise DatabaseConnectionClosedWithDatabase() + self._target_db_instance = None + self._target_db_instance_connection_id = None + + if self._target_db_instance is not None: + return self._target_db_instance + + self._target_db_instance = CustomPeeweeDB( + self.target_db, + user=self.target_db_user, + password=self.target_db_password, + host=self.target_db_host, + port=self.target_db_port, + ) + self._target_db_instance.connect() + # Set session wait timeout to 4 hours [EXPERIMENTAL] + self._target_db_instance.execute_sql("SET SESSION wait_timeout = 14400;") + # Fetch the connection id + self._target_db_instance_connection_id = int( + self._target_db_instance.execute_sql("SELECT CONNECTION_ID();").fetchone()[0] + ) + return self._target_db_instance + + def _get_target_db_for_myisam(self) -> CustomPeeweeDB: + if self._target_db_instance_for_myisam is not None: + if not is_db_connection_usable(self._target_db_instance_for_myisam): + raise DatabaseConnectionClosedWithDatabase() + return self._target_db_instance_for_myisam + + self._target_db_instance_for_myisam = CustomPeeweeDB( + self.target_db, + user=self.target_db_user, + password=self.target_db_password, + host=self.target_db_host, + port=self.target_db_port, + autocommit=False, + ) + self._target_db_instance_for_myisam.connect() + # Set session wait timeout to 4 hours [EXPERIMENTAL] + self._target_db_instance_for_myisam.execute_sql("SET SESSION wait_timeout = 14400;") + # Fetch the connection id + self._target_db_instance_for_myisam_connection_id = int( + self._target_db_instance_for_myisam.execute_sql("SELECT CONNECTION_ID();").fetchone()[0] + ) + return self._target_db_instance_for_myisam + + def _close_db_connections(self): + if self._target_db_instance is not None: + with suppress(Exception): + self._target_db_instance.close() + if self._target_db_instance_for_myisam is not None: + with suppress(Exception): + self._target_db_instance_for_myisam.close() + + def _kill_other_active_db_connections(self): + current_connection_ids = [] + if self._target_db_instance is not None: + current_connection_ids.append(self._target_db_instance_connection_id) + if self._target_db_instance_for_myisam is not None: + current_connection_ids.append(self._target_db_instance_for_myisam_connection_id) + + if not current_connection_ids: + return + + kill_other_db_connections(self._target_db_instance, current_connection_ids) + + def __del__(self): + self._close_db_connections() diff --git a/agent/docker_cache_utils.py b/agent/docker_cache_utils.py index 0ad1b56..95bc7e8 100644 --- a/agent/docker_cache_utils.py +++ b/agent/docker_cache_utils.py @@ -1,259 +1,259 @@ -from __future__ import annotations - -# Code below copied mostly verbatim from jcloud, this is tentative and -# will be removed once build code has been moved out of jcloud. -# -# Primary source: -# http://git.jingrow.com:3000/jingrow/jcloud/blob/40859becf2976a3b6a5ac0ff79e2dff8cd2c46af/jcloud/jcloud/doctype/deploy_candidate/cache_utils.py -import os -import platform -import random -import re -import shlex -import shutil -import subprocess -from datetime import datetime -from pathlib import Path -from textwrap import dedent -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import TypedDict - - class CommandOutput(TypedDict): - cwd: str - image_tag: str - returncode: int - output: str - - -def copy_file_from_docker_cache( - container_source: str, - host_dest: str = ".", - cache_target: str = "/home/jingrow/.cache", -) -> CommandOutput: - """ - Function is used to copy files from docker cache i.e. `cache_target/container_source` - to the host system i.e `host_dest`. - - This function is required cause cache files may be available only during docker build. - - This works by: - - copy the file from mount cache (image) to another_folder (image) - - create a container from image - - copy file from another_folder (container) to host system (using docker cp) - - remove container and then image - """ - filename = Path(container_source).name - container_dest_dirpath = Path(cache_target).parent / "container_dest" - container_dest_filepath = container_dest_dirpath / filename - command = f"mkdir -p {container_dest_dirpath} && " + f"cp {container_source} {container_dest_filepath}" - output = run_command_in_docker_cache( - command, - cache_target, - False, - ) - - if output["returncode"] == 0: - container_id = create_container(output["image_tag"]) - copy_file_from_container( - container_id, - container_dest_filepath, - Path(host_dest), - ) - remove_container(container_id) - - run_image_rm(output["image_tag"]) - return output - - -def run_command_in_docker_cache( - command: str = "ls -A", - cache_target: str = "/home/jingrow/.cache", - remove_image: bool = True, -) -> CommandOutput: - """ - This function works by capturing the output of the given `command` - by running it in the cache dir (`cache_target`) while building a - dummy image. - - The primary purpose is to check the contents of the mounted cache. It's - an incredibly hacky way to achieve this, but afaik the only one. - - Note: The `ARG CACHE_BUST=1` line is used to cause layer cache miss - while running `command` at `cache_target`. This is achieved by changing - `CACHE_BUST` value every run. - - Warning: Takes time to run, use judiciously. - """ - dockerfile = get_cache_check_dockerfile( - command, - cache_target, - ) - df_path = prep_dockerfile_path(dockerfile) - return run_build_command(df_path, remove_image) - - -def get_cache_check_dockerfile(command: str, cache_target: str) -> str: - """ - Note: Mount cache is identified by different attributes, hence it should - be the same as the Dockerfile else it will always result in a cache miss. - - Ref: https://docs.docker.com/engine/reference/builder/#run---mounttypecache - """ - df = f""" - FROM ubuntu:20.04 - ARG CACHE_BUST=1 - WORKDIR {cache_target} - RUN --mount=type=cache,target={cache_target},uid=1000,gid=1000 {command} - """ - return dedent(df).strip() - - -def create_container(image_tag: str) -> str: - args = shlex.split(f"docker create --platform linux/amd64 {image_tag}") - return subprocess.run( - args, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ).stdout.strip() - - -def copy_file_from_container( - container_id: str, - container_filepath: Path, - host_dest: Path, -): - container_source = f"{container_id}:{container_filepath}" - args = ["docker", "cp", container_source, host_dest.as_posix()] - proc = subprocess.run( - args, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - - if not proc.returncode: - print(f"file copied:\n- from {container_source}\n- to {host_dest.absolute().as_posix()}") - else: - print(proc.stdout) - - -def remove_container(container_id: str) -> str: - args = shlex.split(f"docker rm -v {container_id}") - return subprocess.run( - args, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ).stdout - - -def prep_dockerfile_path(dockerfile: str) -> Path: - dir = Path("cache_check_dockerfile_dir") - if dir.is_dir(): - shutil.rmtree(dir) - - dir.mkdir() - df_path = dir / "Dockerfile" - with open(df_path, "w") as df: - df.write(dockerfile) - - return df_path - - -def run_build_command(df_path: Path, remove_image: bool) -> CommandOutput: - command, image_tag = get_cache_check_build_command() - env = os.environ.copy() - env["DOCKER_BUILDKIT"] = "1" - env["BUILDKIT_PROGRESS"] = "plain" - - output = subprocess.run( - shlex.split(command), - env=env, - cwd=df_path.parent, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - if remove_image: - run_image_rm(image_tag) - return dict( - cwd=df_path.parent.absolute().as_posix(), - image_tag=image_tag, - returncode=output.returncode, - output=strip_build_output(output.stdout), - ) - - -def get_cache_check_build_command() -> tuple[str, str]: - command = "docker build" - if platform.machine() == "arm64" and platform.system() == "Darwin" and platform.processor() == "arm": - command += "x build --platform linux/amd64" - - now_ts = datetime.timestamp(datetime.today()) - command += f" --build-arg CACHE_BUST={now_ts}" - - image_tag = f"cache_check:id-{random.getrandbits(40):x}" - command += f" --tag {image_tag} ." - return command, image_tag - - -def run_image_rm(image_tag: str): - command = f"docker image rm {image_tag}" - subprocess.run( - shlex.split(command), - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - -def strip_build_output(stdout: str) -> str: - output = [] - is_output = False - - line_rx = re.compile(r"^#\d+\s\d+\.\d+\s") - done_rx = re.compile(r"^#\d+\sDONE\s\d+\.\d+s$") - - for line in stdout.split("\n"): - if is_output and (m := line_rx.match(line)): - start = m.end() - output.append(line[start:]) - elif is_output and done_rx.search(line): - break - elif "--mount=type=cache,target=" in line: - is_output = True - return "\n".join(output) - - -def get_cached_apps() -> dict[str, list[str]]: - result = run_command_in_docker_cache( - command="ls -A bench/apps", - cache_target="/home/jingrow/.cache", - ) - - apps = dict() - if result["returncode"] != 0: - return apps - - for line in result["output"].split("\n"): - # File Name: app_name-cache_key.ext - splits = line.split("-", 1) - if len(splits) != 2: - continue - - app_name, suffix = splits - suffix_splits = suffix.split(".", 1) - if len(suffix_splits) != 2 or suffix_splits[1] not in ["tar", "tgz"]: - continue - - if app_name not in apps: - apps[app_name] = [] - - app_hash = suffix_splits[0] - apps[app_name].append(app_hash) - return apps +from __future__ import annotations + +# Code below copied mostly verbatim from jcloud, this is tentative and +# will be removed once build code has been moved out of jcloud. +# +# Primary source: +# http://git.jingrow.com/jingrow/jcloud/blob/40859becf2976a3b6a5ac0ff79e2dff8cd2c46af/jcloud/jcloud/doctype/deploy_candidate/cache_utils.py +import os +import platform +import random +import re +import shlex +import shutil +import subprocess +from datetime import datetime +from pathlib import Path +from textwrap import dedent +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import TypedDict + + class CommandOutput(TypedDict): + cwd: str + image_tag: str + returncode: int + output: str + + +def copy_file_from_docker_cache( + container_source: str, + host_dest: str = ".", + cache_target: str = "/home/jingrow/.cache", +) -> CommandOutput: + """ + Function is used to copy files from docker cache i.e. `cache_target/container_source` + to the host system i.e `host_dest`. + + This function is required cause cache files may be available only during docker build. + + This works by: + - copy the file from mount cache (image) to another_folder (image) + - create a container from image + - copy file from another_folder (container) to host system (using docker cp) + - remove container and then image + """ + filename = Path(container_source).name + container_dest_dirpath = Path(cache_target).parent / "container_dest" + container_dest_filepath = container_dest_dirpath / filename + command = f"mkdir -p {container_dest_dirpath} && " + f"cp {container_source} {container_dest_filepath}" + output = run_command_in_docker_cache( + command, + cache_target, + False, + ) + + if output["returncode"] == 0: + container_id = create_container(output["image_tag"]) + copy_file_from_container( + container_id, + container_dest_filepath, + Path(host_dest), + ) + remove_container(container_id) + + run_image_rm(output["image_tag"]) + return output + + +def run_command_in_docker_cache( + command: str = "ls -A", + cache_target: str = "/home/jingrow/.cache", + remove_image: bool = True, +) -> CommandOutput: + """ + This function works by capturing the output of the given `command` + by running it in the cache dir (`cache_target`) while building a + dummy image. + + The primary purpose is to check the contents of the mounted cache. It's + an incredibly hacky way to achieve this, but afaik the only one. + + Note: The `ARG CACHE_BUST=1` line is used to cause layer cache miss + while running `command` at `cache_target`. This is achieved by changing + `CACHE_BUST` value every run. + + Warning: Takes time to run, use judiciously. + """ + dockerfile = get_cache_check_dockerfile( + command, + cache_target, + ) + df_path = prep_dockerfile_path(dockerfile) + return run_build_command(df_path, remove_image) + + +def get_cache_check_dockerfile(command: str, cache_target: str) -> str: + """ + Note: Mount cache is identified by different attributes, hence it should + be the same as the Dockerfile else it will always result in a cache miss. + + Ref: https://docs.docker.com/engine/reference/builder/#run---mounttypecache + """ + df = f""" + FROM ubuntu:20.04 + ARG CACHE_BUST=1 + WORKDIR {cache_target} + RUN --mount=type=cache,target={cache_target},uid=1000,gid=1000 {command} + """ + return dedent(df).strip() + + +def create_container(image_tag: str) -> str: + args = shlex.split(f"docker create --platform linux/amd64 {image_tag}") + return subprocess.run( + args, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout.strip() + + +def copy_file_from_container( + container_id: str, + container_filepath: Path, + host_dest: Path, +): + container_source = f"{container_id}:{container_filepath}" + args = ["docker", "cp", container_source, host_dest.as_posix()] + proc = subprocess.run( + args, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + if not proc.returncode: + print(f"file copied:\n- from {container_source}\n- to {host_dest.absolute().as_posix()}") + else: + print(proc.stdout) + + +def remove_container(container_id: str) -> str: + args = shlex.split(f"docker rm -v {container_id}") + return subprocess.run( + args, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout + + +def prep_dockerfile_path(dockerfile: str) -> Path: + dir = Path("cache_check_dockerfile_dir") + if dir.is_dir(): + shutil.rmtree(dir) + + dir.mkdir() + df_path = dir / "Dockerfile" + with open(df_path, "w") as df: + df.write(dockerfile) + + return df_path + + +def run_build_command(df_path: Path, remove_image: bool) -> CommandOutput: + command, image_tag = get_cache_check_build_command() + env = os.environ.copy() + env["DOCKER_BUILDKIT"] = "1" + env["BUILDKIT_PROGRESS"] = "plain" + + output = subprocess.run( + shlex.split(command), + env=env, + cwd=df_path.parent, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + if remove_image: + run_image_rm(image_tag) + return dict( + cwd=df_path.parent.absolute().as_posix(), + image_tag=image_tag, + returncode=output.returncode, + output=strip_build_output(output.stdout), + ) + + +def get_cache_check_build_command() -> tuple[str, str]: + command = "docker build" + if platform.machine() == "arm64" and platform.system() == "Darwin" and platform.processor() == "arm": + command += "x build --platform linux/amd64" + + now_ts = datetime.timestamp(datetime.today()) + command += f" --build-arg CACHE_BUST={now_ts}" + + image_tag = f"cache_check:id-{random.getrandbits(40):x}" + command += f" --tag {image_tag} ." + return command, image_tag + + +def run_image_rm(image_tag: str): + command = f"docker image rm {image_tag}" + subprocess.run( + shlex.split(command), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def strip_build_output(stdout: str) -> str: + output = [] + is_output = False + + line_rx = re.compile(r"^#\d+\s\d+\.\d+\s") + done_rx = re.compile(r"^#\d+\sDONE\s\d+\.\d+s$") + + for line in stdout.split("\n"): + if is_output and (m := line_rx.match(line)): + start = m.end() + output.append(line[start:]) + elif is_output and done_rx.search(line): + break + elif "--mount=type=cache,target=" in line: + is_output = True + return "\n".join(output) + + +def get_cached_apps() -> dict[str, list[str]]: + result = run_command_in_docker_cache( + command="ls -A bench/apps", + cache_target="/home/jingrow/.cache", + ) + + apps = dict() + if result["returncode"] != 0: + return apps + + for line in result["output"].split("\n"): + # File Name: app_name-cache_key.ext + splits = line.split("-", 1) + if len(splits) != 2: + continue + + app_name, suffix = splits + suffix_splits = suffix.split(".", 1) + if len(suffix_splits) != 2 or suffix_splits[1] not in ["tar", "tgz"]: + continue + + if app_name not in apps: + apps[app_name] = [] + + app_hash = suffix_splits[0] + apps[app_name].append(app_hash) + return apps diff --git a/agent/templates/agent/nginx.conf.jinja2 b/agent/templates/agent/nginx.conf.jinja2 index 2484d21..4fceb8c 100644 --- a/agent/templates/agent/nginx.conf.jinja2 +++ b/agent/templates/agent/nginx.conf.jinja2 @@ -1,301 +1,301 @@ -## Set a variable to help us decide if we need to add the -## 'Docker-Distribution-Api-Version' header. -## The registry always sets this header. -## In the case of nginx performing auth, the header is unset -## since nginx is auth-ing before proxying. -map $upstream_http_docker_distribution_api_version $docker_distribution_api_version { - '' 'registry/2.0'; -} - -## this is required to proxy Grafana Live WebSocket connections -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - - -server { - listen 443 ssl http2; - server_name {{ name }}; - - ssl_certificate {{ tls_directory }}/fullchain.pem; - ssl_certificate_key {{ tls_directory }}/privkey.pem; - ssl_trusted_certificate {{ tls_directory }}/chain.pem; - - ssl_session_timeout 1d; - ssl_session_cache shared:MozSSL:10m; # about 40000 sessions - ssl_session_tickets off; - - ssl_protocols {{ tls_protocols or 'TLSv1.3' }}; - ssl_prefer_server_ciphers off; - - ssl_stapling on; - ssl_stapling_verify on; - - resolver 1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s; - resolver_timeout 2s; - - # disable any limits to avoid HTTP 413 for large image uploads - client_max_body_size 0; - - # required to avoid HTTP 411: see Issue #1486 (http://git.jingrow.com:3000/moby/moby/issues/1486) - chunked_transfer_encoding on; - - # Allow jcloud signup pages to check browser-proxy latency - {% if jcloud_url -%} - more_set_headers "Access-Control-Allow-Origin: {{ jcloud_url }}"; - {%- endif %} - - location /agent/ { - proxy_http_version 1.1; - proxy_cache_bypass $http_upgrade; - - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - - location /agent/benches/metrics { - return 301 /metrics/rq; - } - - proxy_pass http://127.0.0.1:{{ web_port }}/; - } - - {% if nginx_vts_module_enabled %} - location /status { - auth_basic "NGINX VTS"; - auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; - - vhost_traffic_status_display; - vhost_traffic_status_display_format html; - } - {% endif %} - - location /metrics { - auth_basic "Prometheus"; - auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; - - location /metrics/node { - proxy_pass http://127.0.0.1:9100/metrics; - } - - location /metrics/docker { - proxy_pass http://127.0.0.1:9323/metrics; - } - - location /metrics/cadvisor { - proxy_pass http://127.0.0.1:9338/metrics; - } - - {% if nginx_vts_module_enabled %} - location /metrics/nginx { - vhost_traffic_status_display; - vhost_traffic_status_display_format prometheus; - } - {% endif %} - - location /metrics/mariadb { - proxy_pass http://127.0.0.1:9104/metrics; - } - - location /metrics/mariadb_proxy { - proxy_pass http://127.0.0.1:9104/metrics; - } - - location /metrics/gunicorn { - proxy_pass http://127.0.0.1:9102/metrics; - } - - location /metrics/registry { - proxy_pass http://127.0.0.1:5001/metrics; - } - - location /metrics/prometheus { - proxy_pass http://127.0.0.1:9090/prometheus/metrics; - } - - location /metrics/alertmanager { - proxy_pass http://127.0.0.1:9093/alertmanager/metrics; - } - - location /metrics/blackbox { - proxy_pass http://127.0.0.1:9115/blackbox/metrics; - } - - location /metrics/grafana { - proxy_pass http://127.0.0.1:3000/grafana/metrics; - } - - location /metrics/proxysql { - proxy_pass http://127.0.0.1:6070/metrics; - } - - location /metrics/elasticsearch { - proxy_pass http://127.0.0.1:9114/metrics; - } - - location /metrics/rq { - proxy_pass http://127.0.0.1:{{ web_port }}/benches/metrics; - } - - } - - {% if registry %} - - location /v2/ { - # Do not allow connections from docker 1.5 and earlier - # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents - if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { - return 404; - } - - # To add basic authentication to v2 use auth_basic setting. - auth_basic "Registry realm"; - auth_basic_user_file /home/jingrow/registry/registry.htpasswd; - - ## If $docker_distribution_api_version is empty, the header is not added. - ## See the map directive above where this variable is defined. - add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always; - - add_header Access-Control-Allow-Origin '*'; - add_header Access-Control-Allow-Credentials 'true'; - add_header Access-Control-Allow-Headers 'Authorization, Accept'; - add_header Access-Control-Allow-Methods 'HEAD, GET, OPTIONS, DELETE'; - add_header Access-Control-Expose-Headers 'Docker-Content-Digest'; - - proxy_pass http://127.0.0.1:5000; - proxy_set_header Host $http_host; # required for docker client's sake - proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - } - - location / { - # To add basic authentication to v2 use auth_basic setting. - auth_basic "Registry realm"; - auth_basic_user_file /home/jingrow/registry/registry.htpasswd; - - proxy_pass http://127.0.0.1:6000; - proxy_set_header Host $http_host; # required for docker client's sake - proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - } - - {% elif monitor %} - - location /prometheus { - auth_basic "Monitoring"; - auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; - proxy_pass http://127.0.0.1:9090/prometheus; - proxy_read_timeout 1500; - } - - location /alertmanager { - auth_basic "Monitoring"; - auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; - proxy_pass http://127.0.0.1:9093/alertmanager; - } - - location /blackbox { - auth_basic "Monitoring"; - auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; - proxy_pass http://127.0.0.1:9115/blackbox; - } - - location /grafana { - auth_basic "Grafana UI"; - auth_basic_user_file /home/jingrow/agent/nginx/grafana-ui.htpasswd; - - proxy_pass http://127.0.0.1:3000/grafana; - - location /grafana/metrics { - return 307 https://$host/metrics/grafana; - } - - # Proxy Grafana Live WebSocket connections. - location /grafana/api/live/ { - rewrite ^/grafana/(.*) /$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host $http_host; - proxy_pass http://127.0.0.1:3000/grafana; - } - } - - location / { - return 307 https://$host/grafana; - } - - {% elif log %} - - location /kibana/ { - auth_basic "Kibana"; - auth_basic_user_file /home/jingrow/agent/nginx/kibana.htpasswd; - - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - - - proxy_pass http://127.0.0.1:5601/; - } - - location /elasticsearch/ { - auth_basic "Elasticsearch"; - auth_basic_user_file /home/jingrow/agent/nginx/kibana.htpasswd; - - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - - proxy_pass http://127.0.0.1:9200/; - } - - location / { - return 307 https://$host/kibana; - } - - {% elif analytics %} - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass http://127.0.0.1:8000/; - } - - {% elif trace %} - - location / { - proxy_buffer_size 32k; - proxy_buffers 8 16k; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_pass http://127.0.0.1:9000/; - } - - {% else %} - - location / { - root {{ pages_directory }}; - try_files /home.html /dev/null; - } - - {% endif %} -} +## Set a variable to help us decide if we need to add the +## 'Docker-Distribution-Api-Version' header. +## The registry always sets this header. +## In the case of nginx performing auth, the header is unset +## since nginx is auth-ing before proxying. +map $upstream_http_docker_distribution_api_version $docker_distribution_api_version { + '' 'registry/2.0'; +} + +## this is required to proxy Grafana Live WebSocket connections +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + + +server { + listen 443 ssl http2; + server_name {{ name }}; + + ssl_certificate {{ tls_directory }}/fullchain.pem; + ssl_certificate_key {{ tls_directory }}/privkey.pem; + ssl_trusted_certificate {{ tls_directory }}/chain.pem; + + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + ssl_protocols {{ tls_protocols or 'TLSv1.3' }}; + ssl_prefer_server_ciphers off; + + ssl_stapling on; + ssl_stapling_verify on; + + resolver 1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s; + resolver_timeout 2s; + + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # required to avoid HTTP 411: see Issue #1486 (http://git.jingrow.com/moby/moby/issues/1486) + chunked_transfer_encoding on; + + # Allow jcloud signup pages to check browser-proxy latency + {% if jcloud_url -%} + more_set_headers "Access-Control-Allow-Origin: {{ jcloud_url }}"; + {%- endif %} + + location /agent/ { + proxy_http_version 1.1; + proxy_cache_bypass $http_upgrade; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + location /agent/benches/metrics { + return 301 /metrics/rq; + } + + proxy_pass http://127.0.0.1:{{ web_port }}/; + } + + {% if nginx_vts_module_enabled %} + location /status { + auth_basic "NGINX VTS"; + auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; + + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } + {% endif %} + + location /metrics { + auth_basic "Prometheus"; + auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; + + location /metrics/node { + proxy_pass http://127.0.0.1:9100/metrics; + } + + location /metrics/docker { + proxy_pass http://127.0.0.1:9323/metrics; + } + + location /metrics/cadvisor { + proxy_pass http://127.0.0.1:9338/metrics; + } + + {% if nginx_vts_module_enabled %} + location /metrics/nginx { + vhost_traffic_status_display; + vhost_traffic_status_display_format prometheus; + } + {% endif %} + + location /metrics/mariadb { + proxy_pass http://127.0.0.1:9104/metrics; + } + + location /metrics/mariadb_proxy { + proxy_pass http://127.0.0.1:9104/metrics; + } + + location /metrics/gunicorn { + proxy_pass http://127.0.0.1:9102/metrics; + } + + location /metrics/registry { + proxy_pass http://127.0.0.1:5001/metrics; + } + + location /metrics/prometheus { + proxy_pass http://127.0.0.1:9090/prometheus/metrics; + } + + location /metrics/alertmanager { + proxy_pass http://127.0.0.1:9093/alertmanager/metrics; + } + + location /metrics/blackbox { + proxy_pass http://127.0.0.1:9115/blackbox/metrics; + } + + location /metrics/grafana { + proxy_pass http://127.0.0.1:3000/grafana/metrics; + } + + location /metrics/proxysql { + proxy_pass http://127.0.0.1:6070/metrics; + } + + location /metrics/elasticsearch { + proxy_pass http://127.0.0.1:9114/metrics; + } + + location /metrics/rq { + proxy_pass http://127.0.0.1:{{ web_port }}/benches/metrics; + } + + } + + {% if registry %} + + location /v2/ { + # Do not allow connections from docker 1.5 and earlier + # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents + if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { + return 404; + } + + # To add basic authentication to v2 use auth_basic setting. + auth_basic "Registry realm"; + auth_basic_user_file /home/jingrow/registry/registry.htpasswd; + + ## If $docker_distribution_api_version is empty, the header is not added. + ## See the map directive above where this variable is defined. + add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always; + + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Credentials 'true'; + add_header Access-Control-Allow-Headers 'Authorization, Accept'; + add_header Access-Control-Allow-Methods 'HEAD, GET, OPTIONS, DELETE'; + add_header Access-Control-Expose-Headers 'Docker-Content-Digest'; + + proxy_pass http://127.0.0.1:5000; + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + } + + location / { + # To add basic authentication to v2 use auth_basic setting. + auth_basic "Registry realm"; + auth_basic_user_file /home/jingrow/registry/registry.htpasswd; + + proxy_pass http://127.0.0.1:6000; + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + } + + {% elif monitor %} + + location /prometheus { + auth_basic "Monitoring"; + auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; + proxy_pass http://127.0.0.1:9090/prometheus; + proxy_read_timeout 1500; + } + + location /alertmanager { + auth_basic "Monitoring"; + auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; + proxy_pass http://127.0.0.1:9093/alertmanager; + } + + location /blackbox { + auth_basic "Monitoring"; + auth_basic_user_file /home/jingrow/agent/nginx/grafana.htpasswd; + proxy_pass http://127.0.0.1:9115/blackbox; + } + + location /grafana { + auth_basic "Grafana UI"; + auth_basic_user_file /home/jingrow/agent/nginx/grafana-ui.htpasswd; + + proxy_pass http://127.0.0.1:3000/grafana; + + location /grafana/metrics { + return 307 https://$host/metrics/grafana; + } + + # Proxy Grafana Live WebSocket connections. + location /grafana/api/live/ { + rewrite ^/grafana/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://127.0.0.1:3000/grafana; + } + } + + location / { + return 307 https://$host/grafana; + } + + {% elif log %} + + location /kibana/ { + auth_basic "Kibana"; + auth_basic_user_file /home/jingrow/agent/nginx/kibana.htpasswd; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + + proxy_pass http://127.0.0.1:5601/; + } + + location /elasticsearch/ { + auth_basic "Elasticsearch"; + auth_basic_user_file /home/jingrow/agent/nginx/kibana.htpasswd; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + proxy_pass http://127.0.0.1:9200/; + } + + location / { + return 307 https://$host/kibana; + } + + {% elif analytics %} + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://127.0.0.1:8000/; + } + + {% elif trace %} + + location / { + proxy_buffer_size 32k; + proxy_buffers 8 16k; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_pass http://127.0.0.1:9000/; + } + + {% else %} + + location / { + root {{ pages_directory }}; + try_files /home.html /dev/null; + } + + {% endif %} +} diff --git a/agent/web.py b/agent/web.py index fd40831..6a5c34b 100644 --- a/agent/web.py +++ b/agent/web.py @@ -1,1598 +1,1598 @@ -from __future__ import annotations - -import json -import logging -import os -import sys -import traceback -from base64 import b64decode -from functools import wraps -from typing import TYPE_CHECKING - -from flask import Flask, Response, jsonify, request -from passlib.hash import pbkdf2_sha256 as pbkdf2 -from playhouse.shortcuts import model_to_dict -from redis.exceptions import ConnectionError as RedisConnectionError -from rq.exceptions import NoSuchJobError -from rq.job import Job as RQJob -from rq.job import JobStatus - -from agent.base import AgentException -from agent.builder import ImageBuilder, get_image_build_context_directory -from agent.database import JSONEncoderForSQLQueryResult -from agent.database_physical_backup import DatabasePhysicalBackup -from agent.database_physical_restore import DatabasePhysicalRestore -from agent.database_server import DatabaseServer -from agent.exceptions import BenchNotExistsException, SiteNotExistsException -from agent.job import Job as AgentJob -from agent.job import JobModel, connection -from agent.minio import Minio -from agent.monitor import Monitor -from agent.proxy import Proxy -from agent.proxysql import ProxySQL -from agent.security import Security -from agent.server import Server -from agent.ssh import SSHProxy -from agent.utils import check_installed_pyspy - -if TYPE_CHECKING: - from datetime import datetime, timedelta - from typing import TypedDict - - class ExecuteReturn(TypedDict): - command: str - status: str - start: datetime - end: datetime - duration: timedelta - output: str - directory: str | None - traceback: str | None - returncode: int | None - - -application = Flask(__name__) - - -def validate_bench(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - bench = kwargs.get("bench") - - if bench: - Server().get_bench(bench) - - return fn(*args, **kwargs) - - return wrapper - - -def validate_bench_and_site(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - site = kwargs.get("site") - bench = kwargs.get("bench") - - if bench: - bench_obj = Server().get_bench(bench) - bench_obj.get_site(site) - - return fn(*args, **kwargs) - - return wrapper - - -log = logging.getLogger("werkzeug") -log.handlers = [] - - -@application.before_request -def validate_access_token(): - exempt_endpoints = ["get_metrics"] - if request.endpoint in exempt_endpoints: - return None - - try: - if application.debug: - return None - method, access_token = request.headers["Authorization"].split(" ") - stored_hash = Server().config["access_token"] - if method.lower() == "bearer" and pbkdf2.verify(access_token, stored_hash): - return None - access_token = b64decode(access_token).decode().split(":")[1] - if method.lower() == "basic" and pbkdf2.verify(access_token, stored_hash): - return None - except Exception: - pass - - response = jsonify({"message": "Unauthenticated"}) - response.headers.set("WWW-Authenticate", "Basic") - return response, 401 - - -@application.route("/authentication", methods=["POST"]) -def reset_authentication_token(): - data = request.json - Server().setup_authentication(data["token"]) - return jsonify({"message": "Success"}) - - -""" -POST /benches -{ - "name": "bench-1", - "python": "/usr/bin/python3.6", - "apps": [ - { - "name": "jingrow", - "repo": "http://git.jingrow.com:3000/jingrow/jingrow", - "branch": "version-12", - "hash": "ada803c5b57e489bfbc2dee6292a4bcb3ff69aa0" - }, - { - "name": "jerp", - "repo": "http://git.jingrow.com:3000/jingrow/jerp", - "branch": "version-12", - "hash": "782f45ae5f272173b5daadb493d40cf7ccf131b4" - } - ], - "config": { - "background_workers": 8, - "error_report_email": "test@example.com", - "jingrow_user": "jingrow", - "gunicorn_workers": 16, - "mail_login": "test@example.com", - "mail_password": "test", - "mail_server": "smtp.example.com", - "monitor": 1, - "redis_cache": "redis://localhost:13000", - "redis_queue": "redis://localhost:11000", - "redis_socketio": "redis://localhost:12000", - "server_script_enabled": true, - "socketio_port": 9000, - "webserver_port": 8000 - } -} - -""" - - -@application.route("/ping") -def ping(): - return {"message": "pong"} - - -@application.route("/process-snapshot/") -def get_snapshots(bench_name: str): - server = Server() - bench = server.benches.get(bench_name) - - if not bench: - return {"message": f"No such bench {bench_name}"}, 400 - - if not check_installed_pyspy(server.directory): - return {"message": "PySpy is not installed on this server"}, 400 - - pids = bench.get_worker_pids() - return bench.take_snapshot(pids) - - -@application.route("/ping_job", methods=["POST"]) -def ping_job(): - return { - "job": Server().ping_job(), - } - - -@application.route("/builder/upload/", methods=["POST"]) -def upload_build_context_for_image_builder(dc_name: str): - filename = f"{dc_name}.tar.gz" - filepath = os.path.join(get_image_build_context_directory(), filename) - if os.path.exists(filepath): - os.unlink(filepath) - - build_context_file = request.files["build_context_file"] - build_context_file.save(filepath) - return {"filename": filename} - - -@application.route("/builder/build", methods=["POST"]) -def build_image(): - data = request.json - image_builder = ImageBuilder( - filename=data.get("filename"), - image_repository=data.get("image_repository"), - image_tag=data.get("image_tag"), - no_cache=data.get("no_cache"), - no_push=data.get("no_push"), - registry=data.get("registry"), - platform=data.get("platform", "linux/amd64"), - ) - job = image_builder.run_remote_builder() - return {"job": job} - - -@application.route("/server") -def get_server(): - return Server().dump() - - -@application.route("/server/reload", methods=["POST"]) -def restart_nginx(): - job = Server().restart_nginx() - return {"job": job} - - -@application.route("/proxy/reload", methods=["POST"]) -def reload_nginx(): - job = Proxy().reload_nginx_job() - return {"job": job} - - -@application.route("/server/status", methods=["POST"]) -def get_server_status(): - data = request.json - return Server().status(data["mariadb_root_password"]) - - -@application.route("/server/cleanup", methods=["POST"]) -def cleanup_unused_files(): - job = Server().cleanup_unused_files() - return {"job": job} - - -@application.route("/benches") -def get_benches(): - return {name: bench.dump() for name, bench in Server().benches.items()} - - -@application.route("/benches/metrics") -def get_metrics(): - from agent.exporter import get_metrics - - benches_metrics = [] - server = Server() - - for name, bench in server.benches.items(): - rq_port = bench.bench_config.get("rq_port") - if rq_port is not None: - try: - metrics = get_metrics(name, rq_port) - benches_metrics.append(metrics) - except RedisConnectionError as e: - # This is to specifically catch the error on old benches that had their - # configs updated to render rq_port but the container doesn't actually - # expose the rq_port - log.error(f"Failed to get metrics for {name} on port {rq_port}: {e}") - - return Response(benches_metrics, mimetype="text/plain") - - -@application.route("/benches/") -@validate_bench -def get_bench(bench): - return Server().benches[bench].dump() - - -@application.route("/benches//metrics", methods=["GET"]) -def get_bench_metrics(bench_str): - from agent.exporter import get_metrics - - bench = Server().benches[bench_str] - rq_port = bench.bench_config.get("rq_port") - if rq_port: - try: - res = get_metrics(bench_str, rq_port) - except RedisConnectionError as e: - # This is to specifically catch the error on old benches that had their - # configs updated to render rq_port but the container doesn't actually - # expose the rq_port - log.error(f"Failed to get metrics for {bench_str} on port {rq_port}: {e}") - else: - return Response(res, mimetype="text/plain") - - return Response("Unavailable", status=400, mimetype="text/plain") - - -@application.route("/benches//info", methods=["POST", "GET"]) -@validate_bench -def fetch_sites_info(bench): - data = request.json - since = data.get("since") if data else None - return Server().benches[bench].fetch_sites_info(since=since) - - -@application.route("/benches//analytics", methods=["GET"]) -@validate_bench -def fetch_sites_analytics(bench): - return Server().benches[bench].fetch_sites_analytics() - - -@application.route("/benches//sites") -@validate_bench -def get_sites(bench): - sites = Server().benches[bench].sites - return {name: site.dump() for name, site in sites.items()} - - -@application.route("/benches//apps") -@validate_bench -def get_bench_apps(bench): - apps = Server().benches[bench].apps - return {name: site.dump() for name, site in apps.items()} - - -@application.route("/benches//config") -@validate_bench -def get_config(bench): - return Server().benches[bench].config - - -@application.route("/benches//status", methods=["GET"]) -@validate_bench -def get_bench_status(bench): - return Server().benches[bench].status() - - -@application.route("/benches//logs") -@validate_bench -def get_bench_logs(bench): - return jsonify(Server().benches[bench].logs) - - -@application.route("/benches//logs/") -@validate_bench -def get_bench_log(bench, log): - return {log: Server().benches[bench].retrieve_log(log)} - - -@application.route("/benches//sites/") -@validate_bench -def get_site(bench, site): - return Server().benches[bench].sites[site].dump() - - -@application.route("/benches//sites//logs") -@validate_bench_and_site -def get_logs(bench, site): - return jsonify(Server().benches[bench].sites[site].logs) - - -@application.route("/benches//sites//logs/") -@validate_bench_and_site -def get_log(bench, site, log): - return {log: Server().benches[bench].sites[site].retrieve_log(log)} - - -@application.route("/security/ssh_session_logs") -def get_ssh_session_logs(): - return {"logs": Security().ssh_session_logs} - - -@application.route("/security/retrieve_ssh_session_log/") -def retrieve_ssh_session_log(filename): - return {"log_details": Security().retrieve_ssh_session_log(filename)} - - -@application.route("/benches//sites//sid", methods=["GET", "POST"]) -@validate_bench_and_site -def get_site_sid(bench, site): - data = request.json or {} - user = data.get("user") or "Administrator" - return {"sid": Server().benches[bench].sites[site].sid(user=user)} - - -@application.route("/benches", methods=["POST"]) -def new_bench(): - data = request.json - job = Server().new_bench(**data) - return {"job": job} - - -@application.route("/benches//archive", methods=["POST"]) -def archive_bench(bench): - job = Server().archive_bench(bench) - return {"job": job} - - -@application.route("/benches//restart", methods=["POST"]) -@validate_bench -def restart_bench(bench): - data = request.json - job = Server().benches[bench].restart_job(**data) - return {"job": job} - - -@application.route("/benches//limits", methods=["POST"]) -def update_bench_limits(bench): - data = request.json - job = Server().benches[bench].force_update_limits(**data) - return {"job": job} - - -@application.route("/benches//rebuild", methods=["POST"]) -def rebuild_bench(bench): - job = Server().benches[bench].rebuild_job() - return {"job": job} - - -""" -POST /benches/bench-1/sites -{ - "name": "test.jingrow.cloud", - "mariadb_root_password": "root", - "admin_password": "admin", - "apps": ["jingrow", "jcloud"], - "config": { - "monitor": 1, - } -} - -""" - - -@application.route("/benches//sites", methods=["POST"]) -@validate_bench -def new_site(bench): - data = request.json - job = ( - Server() - .benches[bench] - .new_site( - data["name"], - data["config"], - data["apps"], - data["mariadb_root_password"], - data["admin_password"], - create_user=data.get("create_user"), - ) - ) - return {"job": job} - - -@application.route("/benches//sites/restore", methods=["POST"]) -@validate_bench -def new_site_from_backup(bench): - data = request.json - - job = ( - Server() - .benches[bench] - .new_site_from_backup( - data["name"], - data["config"], - data["apps"], - data["mariadb_root_password"], - data["admin_password"], - data["site_config"], - data["database"], - data.get("public"), - data.get("private"), - data.get("skip_failing_patches", False), - ) - ) - return {"job": job} - - -@application.route("/benches//sites//restore", methods=["POST"]) -@validate_bench_and_site -def restore_site(bench, site): - data = request.json - - job = ( - Server() - .benches[bench] - .sites[site] - .restore_job( - data["apps"], - data["mariadb_root_password"], - data["admin_password"], - data["database"], - data.get("public"), - data.get("private"), - data.get("skip_failing_patches", False), - ) - ) - return {"job": job} - - -@application.route("/benches//sites//reinstall", methods=["POST"]) -@validate_bench_and_site -def reinstall_site(bench, site): - data = request.json - job = ( - Server() - .benches[bench] - .sites[site] - .reinstall_job(data["mariadb_root_password"], data["admin_password"]) - ) - return {"job": job} - - -@application.route("/benches//sites//rename", methods=["POST"]) -@validate_bench_and_site -def rename_site(bench, site): - data = request.json - job = ( - Server() - .benches[bench] - .rename_site_job(site, data["new_name"], data.get("create_user"), data.get("config")) - ) - return {"job": job} - - -@application.route("/benches//sites//create-user", methods=["POST"]) -@validate_bench_and_site -def create_user(bench, site): - data = request.json - email = data.get("email") - first_name = data.get("first_name") - last_name = data.get("last_name") - password = data.get("password") - job = ( - Server() - .benches[bench] - .create_user( - site, - email=email, - first_name=first_name, - last_name=last_name, - password=password, - ) - ) - return {"job": job} - - -@application.route( - "/benches//sites//complete-setup-wizard", - methods=["POST"], -) -@validate_bench_and_site -def complete_setup_wizard(bench, site): - data = request.json - job = Server().benches[bench].complete_setup_wizard(site, data) - return {"job": job} - - -@application.route("/benches//sites//optimize", methods=["POST"]) -def optimize_tables(bench, site): - job = Server().benches[bench].sites[site].optimize_tables_job() - return {"job": job} - - -@application.route("/benches//sites//apps", methods=["POST"]) -@validate_bench_and_site -def install_app_site(bench, site): - data = request.json - job = Server().benches[bench].sites[site].install_app_job(data["name"]) - return {"job": job} - - -@application.route( - "/benches//sites//apps/", - methods=["DELETE"], -) -@validate_bench_and_site -def uninstall_app_site(bench, site, app): - job = Server().benches[bench].sites[site].uninstall_app_job(app) - return {"job": job} - - -@application.route("/benches//sites//jerp", methods=["POST"]) -@validate_bench_and_site -def setup_jerp(bench, site): - data = request.json - job = Server().benches[bench].sites[site].setup_jerp(data["user"], data["config"]) - return {"job": job} - - -@application.route("/benches//monitor", methods=["POST"]) -@validate_bench -def fetch_monitor_data(bench): - return {"data": Server().benches[bench].fetch_monitor_data()} - - -@application.route("/benches//sites//status", methods=["GET"]) -@validate_bench_and_site -def fetch_site_status(bench, site): - return {"data": Server().benches[bench].sites[site].fetch_site_status()} - - -@application.route("/benches//sites//info", methods=["GET"]) -@validate_bench_and_site -def fetch_site_info(bench, site): - return {"data": Server().benches[bench].sites[site].fetch_site_info()} - - -@application.route("/benches//sites//analytics", methods=["GET"]) -@validate_bench_and_site -def fetch_site_analytics(bench, site): - return {"data": Server().benches[bench].sites[site].fetch_site_analytics()} - - -@application.route("/benches//sites//backup", methods=["POST"]) -@validate_bench_and_site -def backup_site(bench, site): - data = request.json or {} - with_files = data.get("with_files") - offsite = data.get("offsite") - - job = Server().benches[bench].sites[site].backup_job(with_files, offsite) - return {"job": job} - - -@application.route( - "/benches//sites//database/schema", - methods=["POST"], -) -@validate_bench_and_site -def fetch_database_table_schema(bench, site): - data = request.json or {} - include_table_size = data.get("include_table_size", False) - include_index_info = data.get("include_index_info", False) - job = ( - Server() - .benches[bench] - .sites[site] - .fetch_database_table_schema( - include_table_size=include_table_size, - include_index_info=include_index_info, - ) - ) - return {"job": job} - - -@application.route( - "/benches//sites//database/query/execute", - methods=["POST"], -) -@validate_bench_and_site -def run_sql(bench, site): - query = request.json.get("query") - commit = request.json.get("commit") or False - as_dict = request.json.get("as_dict") or False - return Response( - json.dumps( - Server().benches[bench].sites[site].run_sql_query(query, commit, as_dict), - cls=JSONEncoderForSQLQueryResult, - ), - mimetype="application/json", - ) - - -@application.route( - "/benches//sites//database/analyze-slow-queries", methods=["POST"] -) -@validate_bench_and_site -def analyze_slow_queries(bench: str, site: str): - queries = request.json["queries"] - mariadb_root_password = request.json["mariadb_root_password"] - - job = Server().benches[bench].sites[site].analyze_slow_queries_job(queries, mariadb_root_password) - return {"job": job} - - -@application.route( - "/benches//sites//database/performance-report", methods=["POST"] -) -def database_performance_report(bench, site): - data = request.json - result = ( - Server() - .benches[bench] - .sites[site] - .fetch_summarized_database_performance_report(data["mariadb_root_password"]) - ) - return jsonify(json.loads(json.dumps(result, cls=JSONEncoderForSQLQueryResult))) - - -@application.route("/benches//sites//database/processes", methods=["GET", "POST"]) -def database_process_list(bench, site): - data = request.json - return jsonify( - Server().benches[bench].sites[site].fetch_database_process_list(data["mariadb_root_password"]) - ) - - -@application.route( - "/benches//sites//database/kill-process/", methods=["GET", "POST"] -) -def database_kill_process(bench, site, pid): - data = request.json - Server().benches[bench].sites[site].kill_database_process(pid, data["mariadb_root_password"]) - return {} - - -@application.route("/benches//sites//database/users", methods=["POST"]) -@validate_bench_and_site -def create_database_user(bench, site): - data = request.json - job = ( - Server() - .benches[bench] - .sites[site] - .create_database_user_job(data["username"], data["password"], data["mariadb_root_password"]) - ) - return {"job": job} - - -@application.route( - "/benches//sites//database/users/", - methods=["DELETE"], -) -@validate_bench_and_site -def remove_database_user(bench, site, db_user): - data = request.json - job = Server().benches[bench].sites[site].remove_database_user_job(db_user, data["mariadb_root_password"]) - return {"job": job} - - -@application.route( - "/benches//sites//database/users//permissions", - methods=["POST"], -) -@validate_bench_and_site -def update_database_permissions(bench, site, db_user): - data = request.json - job = ( - Server() - .benches[bench] - .sites[site] - .modify_database_user_permissions_job( - db_user, - data["mode"], - data.get("permissions", {}), - data["mariadb_root_password"], - ) - ) - return {"job": job} - - -@application.route( - "/benches//sites//migrate", - methods=["POST"], -) -@validate_bench_and_site -def migrate_site(bench, site): - data = request.json - job = ( - Server() - .benches[bench] - .sites[site] - .migrate_job( - skip_failing_patches=data.get("skip_failing_patches", False), - activate=data.get("activate", True), - ) - ) - return {"job": job} - - -@application.route( - "/benches//sites//cache", - methods=["DELETE"], -) -@validate_bench_and_site -def clear_site_cache(bench, site): - job = Server().benches[bench].sites[site].clear_cache_job() - return {"job": job} - - -@application.route( - "/benches//sites//activate", - methods=["POST"], -) -@validate_bench_and_site -def activate_site(bench, site): - job = Server().activate_site_job(site, bench) - return {"job": job} - - -@application.route( - "/benches//sites//deactivate", - methods=["POST"], -) -def deactivate_site(bench, site): - job = Server().deactivate_site_job(site, bench) - return {"job": job} - - -@application.route( - "/benches//sites//update/migrate", - methods=["POST"], -) -@validate_bench_and_site -def update_site_migrate(bench, site): - data = request.json - job = Server().update_site_migrate_job( - site, - bench, - data["target"], - data.get("activate", True), - data.get("skip_failing_patches", False), - data.get("skip_backups", False), - data.get("before_migrate_scripts", {}), - data.get("skip_search_index", True), - ) - return {"job": job} - - -@application.route("/benches//sites//update/pull", methods=["POST"]) -@validate_bench_and_site -def update_site_pull(bench, site): - data = request.json - job = Server().update_site_pull_job(site, bench, data["target"], data.get("activate", True)) - return {"job": job} - - -@application.route( - "/benches//sites//update/migrate/recover", - methods=["POST"], -) -@validate_bench_and_site -def update_site_recover_migrate(bench, site): - data = request.json - job = Server().update_site_recover_migrate_job( - site, - bench, - data["target"], - data.get("activate", True), - data.get("rollback_scripts", {}), - data.get("restore_touched_tables", True), - ) - return {"job": job} - - -@application.route( - "/benches//sites//update/migrate/restore", - methods=["POST"], -) -@validate_bench_and_site -def restore_site_tables(bench, site): - data = request.json - job = Server().benches[bench].sites[site].restore_site_tables_job(data.get("activate", True)) - return {"job": job} - - -@application.route( - "/benches//sites//update/pull/recover", - methods=["POST"], -) -@validate_bench_and_site -def update_site_recover_pull(bench, site): - data = request.json - job = Server().update_site_recover_pull_job(site, bench, data["target"], data.get("activate", True)) - return {"job": job} - - -@application.route( - "/benches//sites//update/recover", - methods=["POST"], -) -@validate_bench_and_site -def update_site_recover(bench, site): - job = Server().update_site_recover_job(site, bench) - return {"job": job} - - -@application.route("/benches//sites//archive", methods=["POST"]) -@validate_bench -def archive_site(bench, site): - data = request.json - job = Server().benches[bench].archive_site(site, data["mariadb_root_password"], data.get("force")) - return {"job": job} - - -@application.route("/benches//sites//config", methods=["POST"]) -@validate_bench_and_site -def site_update_config(bench, site): - data = request.json - job = Server().benches[bench].sites[site].update_config_job(data["config"], data["remove"]) - return {"job": job} - - -@application.route("/benches//sites//usage", methods=["DELETE"]) -@validate_bench_and_site -def reset_site_usage(bench, site): - job = Server().benches[bench].sites[site].reset_site_usage_job() - return {"job": job} - - -@application.route("/benches//sites//domains", methods=["POST"]) -@validate_bench_and_site -def site_add_domain(bench, site): - data = request.json - job = Server().benches[bench].sites[site].add_domain(data["domain"]) - return {"job": job} - - -@application.route( - "/benches//sites//domains/", - methods=["DELETE"], -) -@validate_bench_and_site -def site_remove_domain(bench, site, domain): - job = Server().benches[bench].sites[site].remove_domain(domain) - return {"job": job} - - -@application.route( - "/benches//sites//describe-database-table", - methods=["POST"], -) -@validate_bench_and_site -def describe_database_table(bench, site): - data = request.json - return { - "data": Server() - .benches[bench] - .sites[site] - .describe_database_table(data["doctype"], data.get("columns")) - } - - -@application.route( - "/benches//sites//add-database-index", - methods=["POST"], -) -@validate_bench_and_site -def add_database_index(bench, site): - data = request.json - job = Server().benches[bench].sites[site].add_database_index(data["doctype"], data.get("columns")) - return {"job": job} - - -@application.route( - "/benches//sites//apps", - methods=["GET"], -) -@validate_bench_and_site -def get_site_apps(bench, site): - return {"data": Server().benches[bench].sites[site].apps} - - -@application.route( - "/benches//sites//credentials", - methods=["POST"], -) -@validate_bench_and_site -def site_create_database_access_credentials(bench, site): - data = request.json - return ( - Server() - .benches[bench] - .sites[site] - .create_database_access_credentials(data["mode"], data["mariadb_root_password"]) - ) - - -@application.route( - "/benches//sites//credentials/revoke", - methods=["POST"], -) -@validate_bench_and_site -def site_revoke_database_access_credentials(bench, site): - data = request.json - return ( - Server() - .benches[bench] - .sites[site] - .revoke_database_access_credentials(data["user"], data["mariadb_root_password"]) - ) - - -@application.route("/benches//config", methods=["POST"]) -@validate_bench -def bench_set_config(bench): - data = request.json - job = Server().benches[bench].update_config_job(**data) - return {"job": job} - - -@application.route("/proxy/hosts", methods=["POST"]) -def proxy_add_host(): - data = request.json - job = Proxy().add_host_job( - data["name"], - data["target"], - data["certificate"], - data.get("skip_reload", False), - ) - return {"job": job} - - -@application.route("/proxy/wildcards", methods=["POST"]) -def proxy_add_wildcard_hosts(): - data = request.json - job = Proxy().add_wildcard_hosts_job(data) - return {"job": job} - - -@application.route("/proxy/hosts/redirects", methods=["POST"]) -def proxy_setup_redirects(): - data = request.json - job = Proxy().setup_redirects_job(data["domains"], data["target"]) - return {"job": job} - - -@application.route("/proxy/hosts/redirects", methods=["DELETE"]) -def proxy_remove_redirects(): - data = request.json - job = Proxy().remove_redirects_job(data["domains"]) - return {"job": job} - - -@application.route("/proxy/hosts/", methods=["DELETE"]) -def proxy_remove_host(host): - job = Proxy().remove_host_job(host) - return {"job": job} - - -@application.route("/proxy/upstreams", methods=["POST"]) -def proxy_add_upstream(): - data = request.json - job = Proxy().add_upstream_job(data["name"]) - return {"job": job} - - -@application.route("/proxy/upstreams", methods=["GET"]) -def get_upstreams(): - return Proxy().upstreams - - -@application.route("/proxy/upstreams//rename", methods=["POST"]) -def proxy_rename_upstream(upstream): - data = request.json - job = Proxy().rename_upstream_job(upstream, data["name"]) - return {"job": job} - - -@application.route("/proxy/upstreams//sites", methods=["POST"]) -def proxy_add_upstream_site(upstream): - data = request.json - job = Proxy().add_site_to_upstream_job(upstream, data["name"], data.get("skip_reload", False)) - return {"job": job} - - -@application.route("/proxy/upstreams//domains", methods=["POST"]) -def proxy_add_upstream_site_domain(upstream): - data = request.json - job = Proxy().add_domain_to_upstream_job(upstream, data["domain"], data.get("skip_reload", False)) - return {"job": job} - - -@application.route( - "/proxy/upstreams//sites/", - methods=["DELETE"], -) -def proxy_remove_upstream_site(upstream, site): - data = request.json - job = Proxy().remove_site_from_upstream_job(upstream, site, data.get("skip_reload", False)) - return {"job": job} - - -@application.route( - "/proxy/upstreams//sites//rename", - methods=["POST"], -) -def proxy_rename_upstream_site(upstream, site): - data = request.json - job = Proxy().rename_site_on_upstream_job( - upstream, - data["domains"], - site, - data["new_name"], - data.get("skip_reload", False), - ) - return {"job": job} - - -@application.route( - "/proxy/upstreams//sites//status", - methods=["POST"], -) -def update_site_status(upstream, site): - data = request.json - job = Proxy().update_site_status_job( - upstream, site, data["status"], data.get("skip_reload", False), data.get("extra_domains", []) - ) - return {"job": job} - - -@application.route("/monitor/rules", methods=["POST"]) -def update_monitor_rules(): - data = request.json - Monitor().update_rules(data["rules"]) - Monitor().update_routes(data["routes"]) - return {} - - -@application.route("/database/physical-backup", methods=["POST"]) -def physical_backup_database(): - data = request.json - job = DatabasePhysicalBackup( - databases=data["databases"], - db_user="root", - db_host=data["private_ip"], - db_password=data["mariadb_root_password"], - site_backup_name=data["site_backup"]["name"], - snapshot_trigger_url=data["site_backup"]["snapshot_trigger_url"], - snapshot_request_key=data["site_backup"]["snapshot_request_key"], - ).create_backup_job() - return {"job": job} - - -@application.route("/database/physical-restore", methods=["POST"]) -def physical_restore_database(): - data = request.json - job = DatabasePhysicalRestore( - backup_db=data["backup_db"], - target_db=data["target_db"], - target_db_root_password=data["target_db_root_password"], - target_db_port=3306, - target_db_host=data["private_ip"], - backup_db_base_directory=data.get("backup_db_base_directory", ""), - restore_specific_tables=data.get("restore_specific_tables", False), - tables_to_restore=data.get("tables_to_restore", []), - ).create_restore_job() - return {"job": job} - - -@application.route("/database/binary/logs") -def get_binary_logs(): - return jsonify(DatabaseServer().binary_logs) - - -@application.route("/database/processes", methods=["POST"]) -def get_database_processes(): - data = request.json - return jsonify(DatabaseServer().processes(**data)) - - -@application.route("/database/variables", methods=["POST"]) -def get_database_variables(): - data = request.json - return jsonify(DatabaseServer().variables(**data)) - - -@application.route("/database/locks", methods=["POST"]) -def get_database_locks(): - data = request.json - return jsonify(DatabaseServer().locks(**data)) - - -@application.route("/database/processes/kill", methods=["POST"]) -def kill_database_processes(): - data = request.json - return jsonify(DatabaseServer().kill_processes(**data)) - - -@application.route("/database/binary/logs/", methods=["POST"]) -def get_binary_log(log): - data = request.json - return jsonify( - DatabaseServer().search_binary_log( - log, - data["database"], - data["start_datetime"], - data["stop_datetime"], - data["search_pattern"], - data["max_lines"], - ) - ) - - -@application.route("/database/stalks") -def get_stalks(): - return jsonify(DatabaseServer().get_stalks()) - - -@application.route("/database/stalks/") -def get_stalk(stalk): - return jsonify(DatabaseServer().get_stalk(stalk)) - - -@application.route("/database/deadlocks", methods=["POST"]) -def get_database_deadlocks(): - data = request.json - return jsonify(DatabaseServer().get_deadlocks(**data)) - - -# TODO can be removed -@application.route("/database/column-stats", methods=["POST"]) -def fetch_column_statistics(): - data = request.json - job = DatabaseServer().fetch_column_stats_job(**data) - return {"job": job} - - -# TODO can be removed -@application.route("/database/explain", methods=["POST"]) -def explain(): - data = request.json - return jsonify(DatabaseServer().explain_query(**data)) - - -@application.route("/ssh/users", methods=["POST"]) -def ssh_add_user(): - data = request.json - - job = SSHProxy().add_user_job( - data["name"], - data["principal"], - data["ssh"], - data["certificate"], - ) - return {"job": job} - - -@application.route("/ssh/users/", methods=["DELETE"]) -def ssh_remove_user(user): - job = SSHProxy().remove_user_job(user) - return {"job": job} - - -@application.route("/proxysql/users", methods=["POST"]) -def proxysql_add_user(): - data = request.json - - job = ProxySQL().add_user_job( - data["username"], - data["password"], - data["database"], - data["max_connections"], - data["backend"], - ) - return {"job": job} - - -@application.route("/proxysql/backends", methods=["POST"]) -def proxysql_add_backend(): - data = request.json - - job = ProxySQL().add_backend_job(data["backend"]) - return {"job": job} - - -@application.route("/proxysql/users/", methods=["DELETE"]) -def proxysql_remove_user(username): - job = ProxySQL().remove_user_job(username) - return {"job": job} - - -def get_status_from_rq(job, redis): - RQ_STATUS_MAP = { - JobStatus.QUEUED: "Pending", - JobStatus.FINISHED: "Success", - JobStatus.FAILED: "Failure", - JobStatus.STARTED: "Running", - JobStatus.DEFERRED: "Pending", - JobStatus.SCHEDULED: "Pending", - JobStatus.STOPPED: "Failure", - JobStatus.CANCELED: "Failure", - } - status = None - try: - rq_status = RQJob.fetch(str(job["id"]), connection=redis).get_status() - status = RQ_STATUS_MAP.get(rq_status) - except NoSuchJobError: - # Handle jobs enqueued before we started setting job_id - pass - return status - - -def to_dict(model): - redis = connection() - if isinstance(model, JobModel): - job = model_to_dict(model, backrefs=True) - status_from_rq = get_status_from_rq(job, redis) - if status_from_rq: - # Override status from JobModel if rq says the job is already ended - TERMINAL_STATUSES = ["Success", "Failure"] - if job["status"] not in TERMINAL_STATUSES and status_from_rq in TERMINAL_STATUSES: - job["status"] = status_from_rq - - job["data"] = json.loads(job["data"]) or {} - job_key = f"agent:job:{job['id']}" - job["commands"] = [json.loads(command) for command in redis.lrange(job_key, 0, -1)] - for step in job["steps"]: - step["data"] = json.loads(step["data"]) or {} - step_key = f"{job_key}:step:{step['id']}" - step["commands"] = [ - json.loads(command) - for command in redis.lrange( - step_key, - 0, - -1, - ) - ] - else: - job = list(map(model_to_dict, model)) - return job - - -@application.route("/jobs") -@application.route("/jobs/") -@application.route("/jobs/") -@application.route("/jobs/status/") -def jobs(id=None, ids=None, status=None): - choices = [x[1] for x in JobModel._meta.fields["status"].choices] - if id: - data = to_dict(JobModel.get(JobModel.id == id)) - elif ids: - ids = ids.split(",") - data = list(map(to_dict, JobModel.select().where(JobModel.id << ids))) - elif status in choices: - data = to_dict(JobModel.select(JobModel.id, JobModel.name).where(JobModel.status == status)) - else: - data = get_jobs(limit=100) - - return jsonify(json.loads(json.dumps(data, default=str))) - - -@application.route("/jobs//cancel", methods=["POST"]) -def cancel_job(id=None): - job = AgentJob(id=id) - job.cancel_or_stop() - data = to_dict(job.model) - return jsonify(json.loads(json.dumps(data, default=str))) - - -def get_jobs(limit: int = 100): - jobs = ( - JobModel.select( - JobModel.id, - JobModel.name, - JobModel.status, - JobModel.agent_job_id, - JobModel.start, - JobModel.end, - ) - .order_by(JobModel.id.desc()) - .limit(limit) - ) - - data = to_dict(jobs) - for job in data: - del job["duration"] - del job["enqueue"] - del job["data"] - - return data - - -@application.route("/agent-jobs") -@application.route("/agent-jobs/") -@application.route("/agent-jobs/") -def agent_jobs(id=None, ids=None): - if id: - job = to_dict(JobModel.get(JobModel.agent_job_id == id)) - return jsonify(json.loads(json.dumps(job, default=str))) - - if ids: - ids = ids.split(",") - job = list(map(to_dict, JobModel.select().where(JobModel.agent_job_id << ids))) - return jsonify(json.loads(json.dumps(job, default=str))) - - jobs = JobModel.select(JobModel.agent_job_id).order_by(JobModel.id.desc()).limit(100) - jobs = [j["agent_job_id"] for j in to_dict(jobs)] - return jsonify(jobs) - - -@application.route("/update", methods=["POST"]) -def update_agent(): - data = request.json - Server().update_agent_web(url=data.get("url"), branch=data.get("branch")) - return {"message": "Success"} - - -@application.route("/version", methods=["GET"]) -def get_version(): - return Server().get_agent_version() - - -@application.route("/minio/users", methods=["POST"]) -def create_minio_user(): - data = request.json - job = Minio().create_subscription( - data["access_key"], - data["secret_key"], - data["policy_name"], - json.dumps(json.loads(data["policy_json"])), - ) - return {"job": job} - - -@application.route("/minio/users//toggle/", methods=["POST"]) -def toggle_minio_user(username, action): - if action == "disable": - job = Minio().disable_user(username) - else: - job = Minio().enable_user(username) - return {"job": job} - - -@application.route("/minio/users/", methods=["DELETE"]) -def remove_minio_user(username): - job = Minio().remove_user(username) - return {"job": job} - - -@application.route( - "/benches//sites//update/saas", - methods=["POST"], -) -@validate_bench_and_site -def update_saas_plan(bench, site): - data = request.json - job = Server().benches[bench].sites[site].update_saas_plan(data["plan"]) - return {"job": job} - - -@application.route( - "/benches//sites//run_after_migrate_steps", - methods=["POST"], -) -@validate_bench_and_site -def run_after_migrate_steps(bench, site): - data = request.json - job = Server().benches[bench].sites[site].run_after_migrate_steps_job(data["admin_password"]) - return {"job": job} - - -@application.route( - "/benches//sites//move_to_bench", - methods=["POST"], -) -@validate_bench_and_site -def move_site_to_bench(bench, site): - data = request.json - job = Server().move_site_to_bench( - site, - bench, - data["target"], - data.get("deactivate", True), - data.get("activate", True), - data.get("skip_failing_patches", False), - ) - return {"job": job} - - -@application.route("/benches//codeserver", methods=["POST"]) -@validate_bench -def setup_code_server(bench): - data = request.json - job = Server().benches[bench].setup_code_server(**data) - - return {"job": job} - - -@application.route("/benches//codeserver/start", methods=["POST"]) -@validate_bench -def start_code_server(bench): - data = request.json - job = Server().benches[bench].start_code_server(**data) - return {"job": job} - - -@application.route("/benches//codeserver/stop", methods=["POST"]) -@validate_bench -def stop_code_server(bench): - job = Server().benches[bench].stop_code_server() - return {"job": job} - - -@application.route("/benches//codeserver/archive", methods=["POST"]) -@validate_bench -def archive_code_server(bench): - job = Server().benches[bench].archive_code_server() - return {"job": job} - - -@application.route("/benches//patch/", methods=["POST"]) -@validate_bench -def patch_app(bench, app): - data = request.json - job = ( - Server() - .benches[bench] - .patch_app( - app, - data["patch"], - data["filename"], - data["build_assets"], - data["revert"], - ) - ) - return {"job": job} - - -@application.errorhandler(Exception) -def all_exception_handler(error): - try: - from sentry_sdk import capture_exception - - capture_exception(error) - except ImportError: - pass - if isinstance(error, AgentException): - return json.loads(json.dumps(error.data, default=str)), 500 - return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 500 - - -@application.route("/benches//docker_execute", methods=["POST"]) -@validate_bench -def docker_execute(bench: str): - data = request.json - _bench = Server().benches[bench] - result: ExecuteReturn = _bench.docker_execute( - command=data.get("command"), - subdir=data.get("subdir"), - non_zero_throw=False, - ) - - result["start"] = result["start"].isoformat() - result["end"] = result["end"].isoformat() - result["duration"] = result["duration"].total_seconds() - return result - - -@application.route("/benches//supervisorctl", methods=["POST"]) -@validate_bench -def call_bench_supervisorctl(bench: str): - data = request.json - _bench = Server().benches[bench] - job = _bench.call_supervisorctl( - data["command"], - data["programs"], - ) - return {"job": job} - - -@application.errorhandler(BenchNotExistsException) -def bench_not_found(e): - return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 404 - - -@application.errorhandler(SiteNotExistsException) -def site_not_found(e): - return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 404 - - -@application.route("/docker_cache_utils/", methods=["POST"]) -def docker_cache_utils(method: str): - from agent.docker_cache_utils import ( - get_cached_apps, - run_command_in_docker_cache, - ) - - if method == "run_command_in_docker_cache": - return run_command_in_docker_cache(**request.json) - - if method == "get_cached_apps": - return get_cached_apps() - - return None - - -@application.route("/benches//update_inplace", methods=["POST"]) -def update_inplace(bench: str): - sites = request.json.get("sites") - apps = request.json.get("apps") - image = request.json.get("image") - _bench = Server().benches[bench] - job = _bench.update_inplace( - sites, - image, - apps, - ) - return {"job": job} - - -@application.route("/benches//recover_update_inplace", methods=["POST"]) -def recover_update_inplace(bench: str): - _bench = Server().benches[bench] - job = _bench.recover_update_inplace( - request.json.get("sites"), - request.json.get("image"), - ) - return {"job": job} +from __future__ import annotations + +import json +import logging +import os +import sys +import traceback +from base64 import b64decode +from functools import wraps +from typing import TYPE_CHECKING + +from flask import Flask, Response, jsonify, request +from passlib.hash import pbkdf2_sha256 as pbkdf2 +from playhouse.shortcuts import model_to_dict +from redis.exceptions import ConnectionError as RedisConnectionError +from rq.exceptions import NoSuchJobError +from rq.job import Job as RQJob +from rq.job import JobStatus + +from agent.base import AgentException +from agent.builder import ImageBuilder, get_image_build_context_directory +from agent.database import JSONEncoderForSQLQueryResult +from agent.database_physical_backup import DatabasePhysicalBackup +from agent.database_physical_restore import DatabasePhysicalRestore +from agent.database_server import DatabaseServer +from agent.exceptions import BenchNotExistsException, SiteNotExistsException +from agent.job import Job as AgentJob +from agent.job import JobModel, connection +from agent.minio import Minio +from agent.monitor import Monitor +from agent.proxy import Proxy +from agent.proxysql import ProxySQL +from agent.security import Security +from agent.server import Server +from agent.ssh import SSHProxy +from agent.utils import check_installed_pyspy + +if TYPE_CHECKING: + from datetime import datetime, timedelta + from typing import TypedDict + + class ExecuteReturn(TypedDict): + command: str + status: str + start: datetime + end: datetime + duration: timedelta + output: str + directory: str | None + traceback: str | None + returncode: int | None + + +application = Flask(__name__) + + +def validate_bench(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + bench = kwargs.get("bench") + + if bench: + Server().get_bench(bench) + + return fn(*args, **kwargs) + + return wrapper + + +def validate_bench_and_site(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + site = kwargs.get("site") + bench = kwargs.get("bench") + + if bench: + bench_obj = Server().get_bench(bench) + bench_obj.get_site(site) + + return fn(*args, **kwargs) + + return wrapper + + +log = logging.getLogger("werkzeug") +log.handlers = [] + + +@application.before_request +def validate_access_token(): + exempt_endpoints = ["get_metrics"] + if request.endpoint in exempt_endpoints: + return None + + try: + if application.debug: + return None + method, access_token = request.headers["Authorization"].split(" ") + stored_hash = Server().config["access_token"] + if method.lower() == "bearer" and pbkdf2.verify(access_token, stored_hash): + return None + access_token = b64decode(access_token).decode().split(":")[1] + if method.lower() == "basic" and pbkdf2.verify(access_token, stored_hash): + return None + except Exception: + pass + + response = jsonify({"message": "Unauthenticated"}) + response.headers.set("WWW-Authenticate", "Basic") + return response, 401 + + +@application.route("/authentication", methods=["POST"]) +def reset_authentication_token(): + data = request.json + Server().setup_authentication(data["token"]) + return jsonify({"message": "Success"}) + + +""" +POST /benches +{ + "name": "bench-1", + "python": "/usr/bin/python3.6", + "apps": [ + { + "name": "jingrow", + "repo": "http://git.jingrow.com/jingrow/jingrow", + "branch": "version-12", + "hash": "ada803c5b57e489bfbc2dee6292a4bcb3ff69aa0" + }, + { + "name": "jerp", + "repo": "http://git.jingrow.com/jingrow/jerp", + "branch": "version-12", + "hash": "782f45ae5f272173b5daadb493d40cf7ccf131b4" + } + ], + "config": { + "background_workers": 8, + "error_report_email": "test@example.com", + "jingrow_user": "jingrow", + "gunicorn_workers": 16, + "mail_login": "test@example.com", + "mail_password": "test", + "mail_server": "smtp.example.com", + "monitor": 1, + "redis_cache": "redis://localhost:13000", + "redis_queue": "redis://localhost:11000", + "redis_socketio": "redis://localhost:12000", + "server_script_enabled": true, + "socketio_port": 9000, + "webserver_port": 8000 + } +} + +""" + + +@application.route("/ping") +def ping(): + return {"message": "pong"} + + +@application.route("/process-snapshot/") +def get_snapshots(bench_name: str): + server = Server() + bench = server.benches.get(bench_name) + + if not bench: + return {"message": f"No such bench {bench_name}"}, 400 + + if not check_installed_pyspy(server.directory): + return {"message": "PySpy is not installed on this server"}, 400 + + pids = bench.get_worker_pids() + return bench.take_snapshot(pids) + + +@application.route("/ping_job", methods=["POST"]) +def ping_job(): + return { + "job": Server().ping_job(), + } + + +@application.route("/builder/upload/", methods=["POST"]) +def upload_build_context_for_image_builder(dc_name: str): + filename = f"{dc_name}.tar.gz" + filepath = os.path.join(get_image_build_context_directory(), filename) + if os.path.exists(filepath): + os.unlink(filepath) + + build_context_file = request.files["build_context_file"] + build_context_file.save(filepath) + return {"filename": filename} + + +@application.route("/builder/build", methods=["POST"]) +def build_image(): + data = request.json + image_builder = ImageBuilder( + filename=data.get("filename"), + image_repository=data.get("image_repository"), + image_tag=data.get("image_tag"), + no_cache=data.get("no_cache"), + no_push=data.get("no_push"), + registry=data.get("registry"), + platform=data.get("platform", "linux/amd64"), + ) + job = image_builder.run_remote_builder() + return {"job": job} + + +@application.route("/server") +def get_server(): + return Server().dump() + + +@application.route("/server/reload", methods=["POST"]) +def restart_nginx(): + job = Server().restart_nginx() + return {"job": job} + + +@application.route("/proxy/reload", methods=["POST"]) +def reload_nginx(): + job = Proxy().reload_nginx_job() + return {"job": job} + + +@application.route("/server/status", methods=["POST"]) +def get_server_status(): + data = request.json + return Server().status(data["mariadb_root_password"]) + + +@application.route("/server/cleanup", methods=["POST"]) +def cleanup_unused_files(): + job = Server().cleanup_unused_files() + return {"job": job} + + +@application.route("/benches") +def get_benches(): + return {name: bench.dump() for name, bench in Server().benches.items()} + + +@application.route("/benches/metrics") +def get_metrics(): + from agent.exporter import get_metrics + + benches_metrics = [] + server = Server() + + for name, bench in server.benches.items(): + rq_port = bench.bench_config.get("rq_port") + if rq_port is not None: + try: + metrics = get_metrics(name, rq_port) + benches_metrics.append(metrics) + except RedisConnectionError as e: + # This is to specifically catch the error on old benches that had their + # configs updated to render rq_port but the container doesn't actually + # expose the rq_port + log.error(f"Failed to get metrics for {name} on port {rq_port}: {e}") + + return Response(benches_metrics, mimetype="text/plain") + + +@application.route("/benches/") +@validate_bench +def get_bench(bench): + return Server().benches[bench].dump() + + +@application.route("/benches//metrics", methods=["GET"]) +def get_bench_metrics(bench_str): + from agent.exporter import get_metrics + + bench = Server().benches[bench_str] + rq_port = bench.bench_config.get("rq_port") + if rq_port: + try: + res = get_metrics(bench_str, rq_port) + except RedisConnectionError as e: + # This is to specifically catch the error on old benches that had their + # configs updated to render rq_port but the container doesn't actually + # expose the rq_port + log.error(f"Failed to get metrics for {bench_str} on port {rq_port}: {e}") + else: + return Response(res, mimetype="text/plain") + + return Response("Unavailable", status=400, mimetype="text/plain") + + +@application.route("/benches//info", methods=["POST", "GET"]) +@validate_bench +def fetch_sites_info(bench): + data = request.json + since = data.get("since") if data else None + return Server().benches[bench].fetch_sites_info(since=since) + + +@application.route("/benches//analytics", methods=["GET"]) +@validate_bench +def fetch_sites_analytics(bench): + return Server().benches[bench].fetch_sites_analytics() + + +@application.route("/benches//sites") +@validate_bench +def get_sites(bench): + sites = Server().benches[bench].sites + return {name: site.dump() for name, site in sites.items()} + + +@application.route("/benches//apps") +@validate_bench +def get_bench_apps(bench): + apps = Server().benches[bench].apps + return {name: site.dump() for name, site in apps.items()} + + +@application.route("/benches//config") +@validate_bench +def get_config(bench): + return Server().benches[bench].config + + +@application.route("/benches//status", methods=["GET"]) +@validate_bench +def get_bench_status(bench): + return Server().benches[bench].status() + + +@application.route("/benches//logs") +@validate_bench +def get_bench_logs(bench): + return jsonify(Server().benches[bench].logs) + + +@application.route("/benches//logs/") +@validate_bench +def get_bench_log(bench, log): + return {log: Server().benches[bench].retrieve_log(log)} + + +@application.route("/benches//sites/") +@validate_bench +def get_site(bench, site): + return Server().benches[bench].sites[site].dump() + + +@application.route("/benches//sites//logs") +@validate_bench_and_site +def get_logs(bench, site): + return jsonify(Server().benches[bench].sites[site].logs) + + +@application.route("/benches//sites//logs/") +@validate_bench_and_site +def get_log(bench, site, log): + return {log: Server().benches[bench].sites[site].retrieve_log(log)} + + +@application.route("/security/ssh_session_logs") +def get_ssh_session_logs(): + return {"logs": Security().ssh_session_logs} + + +@application.route("/security/retrieve_ssh_session_log/") +def retrieve_ssh_session_log(filename): + return {"log_details": Security().retrieve_ssh_session_log(filename)} + + +@application.route("/benches//sites//sid", methods=["GET", "POST"]) +@validate_bench_and_site +def get_site_sid(bench, site): + data = request.json or {} + user = data.get("user") or "Administrator" + return {"sid": Server().benches[bench].sites[site].sid(user=user)} + + +@application.route("/benches", methods=["POST"]) +def new_bench(): + data = request.json + job = Server().new_bench(**data) + return {"job": job} + + +@application.route("/benches//archive", methods=["POST"]) +def archive_bench(bench): + job = Server().archive_bench(bench) + return {"job": job} + + +@application.route("/benches//restart", methods=["POST"]) +@validate_bench +def restart_bench(bench): + data = request.json + job = Server().benches[bench].restart_job(**data) + return {"job": job} + + +@application.route("/benches//limits", methods=["POST"]) +def update_bench_limits(bench): + data = request.json + job = Server().benches[bench].force_update_limits(**data) + return {"job": job} + + +@application.route("/benches//rebuild", methods=["POST"]) +def rebuild_bench(bench): + job = Server().benches[bench].rebuild_job() + return {"job": job} + + +""" +POST /benches/bench-1/sites +{ + "name": "test.jingrow.cloud", + "mariadb_root_password": "root", + "admin_password": "admin", + "apps": ["jingrow", "jcloud"], + "config": { + "monitor": 1, + } +} + +""" + + +@application.route("/benches//sites", methods=["POST"]) +@validate_bench +def new_site(bench): + data = request.json + job = ( + Server() + .benches[bench] + .new_site( + data["name"], + data["config"], + data["apps"], + data["mariadb_root_password"], + data["admin_password"], + create_user=data.get("create_user"), + ) + ) + return {"job": job} + + +@application.route("/benches//sites/restore", methods=["POST"]) +@validate_bench +def new_site_from_backup(bench): + data = request.json + + job = ( + Server() + .benches[bench] + .new_site_from_backup( + data["name"], + data["config"], + data["apps"], + data["mariadb_root_password"], + data["admin_password"], + data["site_config"], + data["database"], + data.get("public"), + data.get("private"), + data.get("skip_failing_patches", False), + ) + ) + return {"job": job} + + +@application.route("/benches//sites//restore", methods=["POST"]) +@validate_bench_and_site +def restore_site(bench, site): + data = request.json + + job = ( + Server() + .benches[bench] + .sites[site] + .restore_job( + data["apps"], + data["mariadb_root_password"], + data["admin_password"], + data["database"], + data.get("public"), + data.get("private"), + data.get("skip_failing_patches", False), + ) + ) + return {"job": job} + + +@application.route("/benches//sites//reinstall", methods=["POST"]) +@validate_bench_and_site +def reinstall_site(bench, site): + data = request.json + job = ( + Server() + .benches[bench] + .sites[site] + .reinstall_job(data["mariadb_root_password"], data["admin_password"]) + ) + return {"job": job} + + +@application.route("/benches//sites//rename", methods=["POST"]) +@validate_bench_and_site +def rename_site(bench, site): + data = request.json + job = ( + Server() + .benches[bench] + .rename_site_job(site, data["new_name"], data.get("create_user"), data.get("config")) + ) + return {"job": job} + + +@application.route("/benches//sites//create-user", methods=["POST"]) +@validate_bench_and_site +def create_user(bench, site): + data = request.json + email = data.get("email") + first_name = data.get("first_name") + last_name = data.get("last_name") + password = data.get("password") + job = ( + Server() + .benches[bench] + .create_user( + site, + email=email, + first_name=first_name, + last_name=last_name, + password=password, + ) + ) + return {"job": job} + + +@application.route( + "/benches//sites//complete-setup-wizard", + methods=["POST"], +) +@validate_bench_and_site +def complete_setup_wizard(bench, site): + data = request.json + job = Server().benches[bench].complete_setup_wizard(site, data) + return {"job": job} + + +@application.route("/benches//sites//optimize", methods=["POST"]) +def optimize_tables(bench, site): + job = Server().benches[bench].sites[site].optimize_tables_job() + return {"job": job} + + +@application.route("/benches//sites//apps", methods=["POST"]) +@validate_bench_and_site +def install_app_site(bench, site): + data = request.json + job = Server().benches[bench].sites[site].install_app_job(data["name"]) + return {"job": job} + + +@application.route( + "/benches//sites//apps/", + methods=["DELETE"], +) +@validate_bench_and_site +def uninstall_app_site(bench, site, app): + job = Server().benches[bench].sites[site].uninstall_app_job(app) + return {"job": job} + + +@application.route("/benches//sites//jerp", methods=["POST"]) +@validate_bench_and_site +def setup_jerp(bench, site): + data = request.json + job = Server().benches[bench].sites[site].setup_jerp(data["user"], data["config"]) + return {"job": job} + + +@application.route("/benches//monitor", methods=["POST"]) +@validate_bench +def fetch_monitor_data(bench): + return {"data": Server().benches[bench].fetch_monitor_data()} + + +@application.route("/benches//sites//status", methods=["GET"]) +@validate_bench_and_site +def fetch_site_status(bench, site): + return {"data": Server().benches[bench].sites[site].fetch_site_status()} + + +@application.route("/benches//sites//info", methods=["GET"]) +@validate_bench_and_site +def fetch_site_info(bench, site): + return {"data": Server().benches[bench].sites[site].fetch_site_info()} + + +@application.route("/benches//sites//analytics", methods=["GET"]) +@validate_bench_and_site +def fetch_site_analytics(bench, site): + return {"data": Server().benches[bench].sites[site].fetch_site_analytics()} + + +@application.route("/benches//sites//backup", methods=["POST"]) +@validate_bench_and_site +def backup_site(bench, site): + data = request.json or {} + with_files = data.get("with_files") + offsite = data.get("offsite") + + job = Server().benches[bench].sites[site].backup_job(with_files, offsite) + return {"job": job} + + +@application.route( + "/benches//sites//database/schema", + methods=["POST"], +) +@validate_bench_and_site +def fetch_database_table_schema(bench, site): + data = request.json or {} + include_table_size = data.get("include_table_size", False) + include_index_info = data.get("include_index_info", False) + job = ( + Server() + .benches[bench] + .sites[site] + .fetch_database_table_schema( + include_table_size=include_table_size, + include_index_info=include_index_info, + ) + ) + return {"job": job} + + +@application.route( + "/benches//sites//database/query/execute", + methods=["POST"], +) +@validate_bench_and_site +def run_sql(bench, site): + query = request.json.get("query") + commit = request.json.get("commit") or False + as_dict = request.json.get("as_dict") or False + return Response( + json.dumps( + Server().benches[bench].sites[site].run_sql_query(query, commit, as_dict), + cls=JSONEncoderForSQLQueryResult, + ), + mimetype="application/json", + ) + + +@application.route( + "/benches//sites//database/analyze-slow-queries", methods=["POST"] +) +@validate_bench_and_site +def analyze_slow_queries(bench: str, site: str): + queries = request.json["queries"] + mariadb_root_password = request.json["mariadb_root_password"] + + job = Server().benches[bench].sites[site].analyze_slow_queries_job(queries, mariadb_root_password) + return {"job": job} + + +@application.route( + "/benches//sites//database/performance-report", methods=["POST"] +) +def database_performance_report(bench, site): + data = request.json + result = ( + Server() + .benches[bench] + .sites[site] + .fetch_summarized_database_performance_report(data["mariadb_root_password"]) + ) + return jsonify(json.loads(json.dumps(result, cls=JSONEncoderForSQLQueryResult))) + + +@application.route("/benches//sites//database/processes", methods=["GET", "POST"]) +def database_process_list(bench, site): + data = request.json + return jsonify( + Server().benches[bench].sites[site].fetch_database_process_list(data["mariadb_root_password"]) + ) + + +@application.route( + "/benches//sites//database/kill-process/", methods=["GET", "POST"] +) +def database_kill_process(bench, site, pid): + data = request.json + Server().benches[bench].sites[site].kill_database_process(pid, data["mariadb_root_password"]) + return {} + + +@application.route("/benches//sites//database/users", methods=["POST"]) +@validate_bench_and_site +def create_database_user(bench, site): + data = request.json + job = ( + Server() + .benches[bench] + .sites[site] + .create_database_user_job(data["username"], data["password"], data["mariadb_root_password"]) + ) + return {"job": job} + + +@application.route( + "/benches//sites//database/users/", + methods=["DELETE"], +) +@validate_bench_and_site +def remove_database_user(bench, site, db_user): + data = request.json + job = Server().benches[bench].sites[site].remove_database_user_job(db_user, data["mariadb_root_password"]) + return {"job": job} + + +@application.route( + "/benches//sites//database/users//permissions", + methods=["POST"], +) +@validate_bench_and_site +def update_database_permissions(bench, site, db_user): + data = request.json + job = ( + Server() + .benches[bench] + .sites[site] + .modify_database_user_permissions_job( + db_user, + data["mode"], + data.get("permissions", {}), + data["mariadb_root_password"], + ) + ) + return {"job": job} + + +@application.route( + "/benches//sites//migrate", + methods=["POST"], +) +@validate_bench_and_site +def migrate_site(bench, site): + data = request.json + job = ( + Server() + .benches[bench] + .sites[site] + .migrate_job( + skip_failing_patches=data.get("skip_failing_patches", False), + activate=data.get("activate", True), + ) + ) + return {"job": job} + + +@application.route( + "/benches//sites//cache", + methods=["DELETE"], +) +@validate_bench_and_site +def clear_site_cache(bench, site): + job = Server().benches[bench].sites[site].clear_cache_job() + return {"job": job} + + +@application.route( + "/benches//sites//activate", + methods=["POST"], +) +@validate_bench_and_site +def activate_site(bench, site): + job = Server().activate_site_job(site, bench) + return {"job": job} + + +@application.route( + "/benches//sites//deactivate", + methods=["POST"], +) +def deactivate_site(bench, site): + job = Server().deactivate_site_job(site, bench) + return {"job": job} + + +@application.route( + "/benches//sites//update/migrate", + methods=["POST"], +) +@validate_bench_and_site +def update_site_migrate(bench, site): + data = request.json + job = Server().update_site_migrate_job( + site, + bench, + data["target"], + data.get("activate", True), + data.get("skip_failing_patches", False), + data.get("skip_backups", False), + data.get("before_migrate_scripts", {}), + data.get("skip_search_index", True), + ) + return {"job": job} + + +@application.route("/benches//sites//update/pull", methods=["POST"]) +@validate_bench_and_site +def update_site_pull(bench, site): + data = request.json + job = Server().update_site_pull_job(site, bench, data["target"], data.get("activate", True)) + return {"job": job} + + +@application.route( + "/benches//sites//update/migrate/recover", + methods=["POST"], +) +@validate_bench_and_site +def update_site_recover_migrate(bench, site): + data = request.json + job = Server().update_site_recover_migrate_job( + site, + bench, + data["target"], + data.get("activate", True), + data.get("rollback_scripts", {}), + data.get("restore_touched_tables", True), + ) + return {"job": job} + + +@application.route( + "/benches//sites//update/migrate/restore", + methods=["POST"], +) +@validate_bench_and_site +def restore_site_tables(bench, site): + data = request.json + job = Server().benches[bench].sites[site].restore_site_tables_job(data.get("activate", True)) + return {"job": job} + + +@application.route( + "/benches//sites//update/pull/recover", + methods=["POST"], +) +@validate_bench_and_site +def update_site_recover_pull(bench, site): + data = request.json + job = Server().update_site_recover_pull_job(site, bench, data["target"], data.get("activate", True)) + return {"job": job} + + +@application.route( + "/benches//sites//update/recover", + methods=["POST"], +) +@validate_bench_and_site +def update_site_recover(bench, site): + job = Server().update_site_recover_job(site, bench) + return {"job": job} + + +@application.route("/benches//sites//archive", methods=["POST"]) +@validate_bench +def archive_site(bench, site): + data = request.json + job = Server().benches[bench].archive_site(site, data["mariadb_root_password"], data.get("force")) + return {"job": job} + + +@application.route("/benches//sites//config", methods=["POST"]) +@validate_bench_and_site +def site_update_config(bench, site): + data = request.json + job = Server().benches[bench].sites[site].update_config_job(data["config"], data["remove"]) + return {"job": job} + + +@application.route("/benches//sites//usage", methods=["DELETE"]) +@validate_bench_and_site +def reset_site_usage(bench, site): + job = Server().benches[bench].sites[site].reset_site_usage_job() + return {"job": job} + + +@application.route("/benches//sites//domains", methods=["POST"]) +@validate_bench_and_site +def site_add_domain(bench, site): + data = request.json + job = Server().benches[bench].sites[site].add_domain(data["domain"]) + return {"job": job} + + +@application.route( + "/benches//sites//domains/", + methods=["DELETE"], +) +@validate_bench_and_site +def site_remove_domain(bench, site, domain): + job = Server().benches[bench].sites[site].remove_domain(domain) + return {"job": job} + + +@application.route( + "/benches//sites//describe-database-table", + methods=["POST"], +) +@validate_bench_and_site +def describe_database_table(bench, site): + data = request.json + return { + "data": Server() + .benches[bench] + .sites[site] + .describe_database_table(data["doctype"], data.get("columns")) + } + + +@application.route( + "/benches//sites//add-database-index", + methods=["POST"], +) +@validate_bench_and_site +def add_database_index(bench, site): + data = request.json + job = Server().benches[bench].sites[site].add_database_index(data["doctype"], data.get("columns")) + return {"job": job} + + +@application.route( + "/benches//sites//apps", + methods=["GET"], +) +@validate_bench_and_site +def get_site_apps(bench, site): + return {"data": Server().benches[bench].sites[site].apps} + + +@application.route( + "/benches//sites//credentials", + methods=["POST"], +) +@validate_bench_and_site +def site_create_database_access_credentials(bench, site): + data = request.json + return ( + Server() + .benches[bench] + .sites[site] + .create_database_access_credentials(data["mode"], data["mariadb_root_password"]) + ) + + +@application.route( + "/benches//sites//credentials/revoke", + methods=["POST"], +) +@validate_bench_and_site +def site_revoke_database_access_credentials(bench, site): + data = request.json + return ( + Server() + .benches[bench] + .sites[site] + .revoke_database_access_credentials(data["user"], data["mariadb_root_password"]) + ) + + +@application.route("/benches//config", methods=["POST"]) +@validate_bench +def bench_set_config(bench): + data = request.json + job = Server().benches[bench].update_config_job(**data) + return {"job": job} + + +@application.route("/proxy/hosts", methods=["POST"]) +def proxy_add_host(): + data = request.json + job = Proxy().add_host_job( + data["name"], + data["target"], + data["certificate"], + data.get("skip_reload", False), + ) + return {"job": job} + + +@application.route("/proxy/wildcards", methods=["POST"]) +def proxy_add_wildcard_hosts(): + data = request.json + job = Proxy().add_wildcard_hosts_job(data) + return {"job": job} + + +@application.route("/proxy/hosts/redirects", methods=["POST"]) +def proxy_setup_redirects(): + data = request.json + job = Proxy().setup_redirects_job(data["domains"], data["target"]) + return {"job": job} + + +@application.route("/proxy/hosts/redirects", methods=["DELETE"]) +def proxy_remove_redirects(): + data = request.json + job = Proxy().remove_redirects_job(data["domains"]) + return {"job": job} + + +@application.route("/proxy/hosts/", methods=["DELETE"]) +def proxy_remove_host(host): + job = Proxy().remove_host_job(host) + return {"job": job} + + +@application.route("/proxy/upstreams", methods=["POST"]) +def proxy_add_upstream(): + data = request.json + job = Proxy().add_upstream_job(data["name"]) + return {"job": job} + + +@application.route("/proxy/upstreams", methods=["GET"]) +def get_upstreams(): + return Proxy().upstreams + + +@application.route("/proxy/upstreams//rename", methods=["POST"]) +def proxy_rename_upstream(upstream): + data = request.json + job = Proxy().rename_upstream_job(upstream, data["name"]) + return {"job": job} + + +@application.route("/proxy/upstreams//sites", methods=["POST"]) +def proxy_add_upstream_site(upstream): + data = request.json + job = Proxy().add_site_to_upstream_job(upstream, data["name"], data.get("skip_reload", False)) + return {"job": job} + + +@application.route("/proxy/upstreams//domains", methods=["POST"]) +def proxy_add_upstream_site_domain(upstream): + data = request.json + job = Proxy().add_domain_to_upstream_job(upstream, data["domain"], data.get("skip_reload", False)) + return {"job": job} + + +@application.route( + "/proxy/upstreams//sites/", + methods=["DELETE"], +) +def proxy_remove_upstream_site(upstream, site): + data = request.json + job = Proxy().remove_site_from_upstream_job(upstream, site, data.get("skip_reload", False)) + return {"job": job} + + +@application.route( + "/proxy/upstreams//sites//rename", + methods=["POST"], +) +def proxy_rename_upstream_site(upstream, site): + data = request.json + job = Proxy().rename_site_on_upstream_job( + upstream, + data["domains"], + site, + data["new_name"], + data.get("skip_reload", False), + ) + return {"job": job} + + +@application.route( + "/proxy/upstreams//sites//status", + methods=["POST"], +) +def update_site_status(upstream, site): + data = request.json + job = Proxy().update_site_status_job( + upstream, site, data["status"], data.get("skip_reload", False), data.get("extra_domains", []) + ) + return {"job": job} + + +@application.route("/monitor/rules", methods=["POST"]) +def update_monitor_rules(): + data = request.json + Monitor().update_rules(data["rules"]) + Monitor().update_routes(data["routes"]) + return {} + + +@application.route("/database/physical-backup", methods=["POST"]) +def physical_backup_database(): + data = request.json + job = DatabasePhysicalBackup( + databases=data["databases"], + db_user="root", + db_host=data["private_ip"], + db_password=data["mariadb_root_password"], + site_backup_name=data["site_backup"]["name"], + snapshot_trigger_url=data["site_backup"]["snapshot_trigger_url"], + snapshot_request_key=data["site_backup"]["snapshot_request_key"], + ).create_backup_job() + return {"job": job} + + +@application.route("/database/physical-restore", methods=["POST"]) +def physical_restore_database(): + data = request.json + job = DatabasePhysicalRestore( + backup_db=data["backup_db"], + target_db=data["target_db"], + target_db_root_password=data["target_db_root_password"], + target_db_port=3306, + target_db_host=data["private_ip"], + backup_db_base_directory=data.get("backup_db_base_directory", ""), + restore_specific_tables=data.get("restore_specific_tables", False), + tables_to_restore=data.get("tables_to_restore", []), + ).create_restore_job() + return {"job": job} + + +@application.route("/database/binary/logs") +def get_binary_logs(): + return jsonify(DatabaseServer().binary_logs) + + +@application.route("/database/processes", methods=["POST"]) +def get_database_processes(): + data = request.json + return jsonify(DatabaseServer().processes(**data)) + + +@application.route("/database/variables", methods=["POST"]) +def get_database_variables(): + data = request.json + return jsonify(DatabaseServer().variables(**data)) + + +@application.route("/database/locks", methods=["POST"]) +def get_database_locks(): + data = request.json + return jsonify(DatabaseServer().locks(**data)) + + +@application.route("/database/processes/kill", methods=["POST"]) +def kill_database_processes(): + data = request.json + return jsonify(DatabaseServer().kill_processes(**data)) + + +@application.route("/database/binary/logs/", methods=["POST"]) +def get_binary_log(log): + data = request.json + return jsonify( + DatabaseServer().search_binary_log( + log, + data["database"], + data["start_datetime"], + data["stop_datetime"], + data["search_pattern"], + data["max_lines"], + ) + ) + + +@application.route("/database/stalks") +def get_stalks(): + return jsonify(DatabaseServer().get_stalks()) + + +@application.route("/database/stalks/") +def get_stalk(stalk): + return jsonify(DatabaseServer().get_stalk(stalk)) + + +@application.route("/database/deadlocks", methods=["POST"]) +def get_database_deadlocks(): + data = request.json + return jsonify(DatabaseServer().get_deadlocks(**data)) + + +# TODO can be removed +@application.route("/database/column-stats", methods=["POST"]) +def fetch_column_statistics(): + data = request.json + job = DatabaseServer().fetch_column_stats_job(**data) + return {"job": job} + + +# TODO can be removed +@application.route("/database/explain", methods=["POST"]) +def explain(): + data = request.json + return jsonify(DatabaseServer().explain_query(**data)) + + +@application.route("/ssh/users", methods=["POST"]) +def ssh_add_user(): + data = request.json + + job = SSHProxy().add_user_job( + data["name"], + data["principal"], + data["ssh"], + data["certificate"], + ) + return {"job": job} + + +@application.route("/ssh/users/", methods=["DELETE"]) +def ssh_remove_user(user): + job = SSHProxy().remove_user_job(user) + return {"job": job} + + +@application.route("/proxysql/users", methods=["POST"]) +def proxysql_add_user(): + data = request.json + + job = ProxySQL().add_user_job( + data["username"], + data["password"], + data["database"], + data["max_connections"], + data["backend"], + ) + return {"job": job} + + +@application.route("/proxysql/backends", methods=["POST"]) +def proxysql_add_backend(): + data = request.json + + job = ProxySQL().add_backend_job(data["backend"]) + return {"job": job} + + +@application.route("/proxysql/users/", methods=["DELETE"]) +def proxysql_remove_user(username): + job = ProxySQL().remove_user_job(username) + return {"job": job} + + +def get_status_from_rq(job, redis): + RQ_STATUS_MAP = { + JobStatus.QUEUED: "Pending", + JobStatus.FINISHED: "Success", + JobStatus.FAILED: "Failure", + JobStatus.STARTED: "Running", + JobStatus.DEFERRED: "Pending", + JobStatus.SCHEDULED: "Pending", + JobStatus.STOPPED: "Failure", + JobStatus.CANCELED: "Failure", + } + status = None + try: + rq_status = RQJob.fetch(str(job["id"]), connection=redis).get_status() + status = RQ_STATUS_MAP.get(rq_status) + except NoSuchJobError: + # Handle jobs enqueued before we started setting job_id + pass + return status + + +def to_dict(model): + redis = connection() + if isinstance(model, JobModel): + job = model_to_dict(model, backrefs=True) + status_from_rq = get_status_from_rq(job, redis) + if status_from_rq: + # Override status from JobModel if rq says the job is already ended + TERMINAL_STATUSES = ["Success", "Failure"] + if job["status"] not in TERMINAL_STATUSES and status_from_rq in TERMINAL_STATUSES: + job["status"] = status_from_rq + + job["data"] = json.loads(job["data"]) or {} + job_key = f"agent:job:{job['id']}" + job["commands"] = [json.loads(command) for command in redis.lrange(job_key, 0, -1)] + for step in job["steps"]: + step["data"] = json.loads(step["data"]) or {} + step_key = f"{job_key}:step:{step['id']}" + step["commands"] = [ + json.loads(command) + for command in redis.lrange( + step_key, + 0, + -1, + ) + ] + else: + job = list(map(model_to_dict, model)) + return job + + +@application.route("/jobs") +@application.route("/jobs/") +@application.route("/jobs/") +@application.route("/jobs/status/") +def jobs(id=None, ids=None, status=None): + choices = [x[1] for x in JobModel._meta.fields["status"].choices] + if id: + data = to_dict(JobModel.get(JobModel.id == id)) + elif ids: + ids = ids.split(",") + data = list(map(to_dict, JobModel.select().where(JobModel.id << ids))) + elif status in choices: + data = to_dict(JobModel.select(JobModel.id, JobModel.name).where(JobModel.status == status)) + else: + data = get_jobs(limit=100) + + return jsonify(json.loads(json.dumps(data, default=str))) + + +@application.route("/jobs//cancel", methods=["POST"]) +def cancel_job(id=None): + job = AgentJob(id=id) + job.cancel_or_stop() + data = to_dict(job.model) + return jsonify(json.loads(json.dumps(data, default=str))) + + +def get_jobs(limit: int = 100): + jobs = ( + JobModel.select( + JobModel.id, + JobModel.name, + JobModel.status, + JobModel.agent_job_id, + JobModel.start, + JobModel.end, + ) + .order_by(JobModel.id.desc()) + .limit(limit) + ) + + data = to_dict(jobs) + for job in data: + del job["duration"] + del job["enqueue"] + del job["data"] + + return data + + +@application.route("/agent-jobs") +@application.route("/agent-jobs/") +@application.route("/agent-jobs/") +def agent_jobs(id=None, ids=None): + if id: + job = to_dict(JobModel.get(JobModel.agent_job_id == id)) + return jsonify(json.loads(json.dumps(job, default=str))) + + if ids: + ids = ids.split(",") + job = list(map(to_dict, JobModel.select().where(JobModel.agent_job_id << ids))) + return jsonify(json.loads(json.dumps(job, default=str))) + + jobs = JobModel.select(JobModel.agent_job_id).order_by(JobModel.id.desc()).limit(100) + jobs = [j["agent_job_id"] for j in to_dict(jobs)] + return jsonify(jobs) + + +@application.route("/update", methods=["POST"]) +def update_agent(): + data = request.json + Server().update_agent_web(url=data.get("url"), branch=data.get("branch")) + return {"message": "Success"} + + +@application.route("/version", methods=["GET"]) +def get_version(): + return Server().get_agent_version() + + +@application.route("/minio/users", methods=["POST"]) +def create_minio_user(): + data = request.json + job = Minio().create_subscription( + data["access_key"], + data["secret_key"], + data["policy_name"], + json.dumps(json.loads(data["policy_json"])), + ) + return {"job": job} + + +@application.route("/minio/users//toggle/", methods=["POST"]) +def toggle_minio_user(username, action): + if action == "disable": + job = Minio().disable_user(username) + else: + job = Minio().enable_user(username) + return {"job": job} + + +@application.route("/minio/users/", methods=["DELETE"]) +def remove_minio_user(username): + job = Minio().remove_user(username) + return {"job": job} + + +@application.route( + "/benches//sites//update/saas", + methods=["POST"], +) +@validate_bench_and_site +def update_saas_plan(bench, site): + data = request.json + job = Server().benches[bench].sites[site].update_saas_plan(data["plan"]) + return {"job": job} + + +@application.route( + "/benches//sites//run_after_migrate_steps", + methods=["POST"], +) +@validate_bench_and_site +def run_after_migrate_steps(bench, site): + data = request.json + job = Server().benches[bench].sites[site].run_after_migrate_steps_job(data["admin_password"]) + return {"job": job} + + +@application.route( + "/benches//sites//move_to_bench", + methods=["POST"], +) +@validate_bench_and_site +def move_site_to_bench(bench, site): + data = request.json + job = Server().move_site_to_bench( + site, + bench, + data["target"], + data.get("deactivate", True), + data.get("activate", True), + data.get("skip_failing_patches", False), + ) + return {"job": job} + + +@application.route("/benches//codeserver", methods=["POST"]) +@validate_bench +def setup_code_server(bench): + data = request.json + job = Server().benches[bench].setup_code_server(**data) + + return {"job": job} + + +@application.route("/benches//codeserver/start", methods=["POST"]) +@validate_bench +def start_code_server(bench): + data = request.json + job = Server().benches[bench].start_code_server(**data) + return {"job": job} + + +@application.route("/benches//codeserver/stop", methods=["POST"]) +@validate_bench +def stop_code_server(bench): + job = Server().benches[bench].stop_code_server() + return {"job": job} + + +@application.route("/benches//codeserver/archive", methods=["POST"]) +@validate_bench +def archive_code_server(bench): + job = Server().benches[bench].archive_code_server() + return {"job": job} + + +@application.route("/benches//patch/", methods=["POST"]) +@validate_bench +def patch_app(bench, app): + data = request.json + job = ( + Server() + .benches[bench] + .patch_app( + app, + data["patch"], + data["filename"], + data["build_assets"], + data["revert"], + ) + ) + return {"job": job} + + +@application.errorhandler(Exception) +def all_exception_handler(error): + try: + from sentry_sdk import capture_exception + + capture_exception(error) + except ImportError: + pass + if isinstance(error, AgentException): + return json.loads(json.dumps(error.data, default=str)), 500 + return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 500 + + +@application.route("/benches//docker_execute", methods=["POST"]) +@validate_bench +def docker_execute(bench: str): + data = request.json + _bench = Server().benches[bench] + result: ExecuteReturn = _bench.docker_execute( + command=data.get("command"), + subdir=data.get("subdir"), + non_zero_throw=False, + ) + + result["start"] = result["start"].isoformat() + result["end"] = result["end"].isoformat() + result["duration"] = result["duration"].total_seconds() + return result + + +@application.route("/benches//supervisorctl", methods=["POST"]) +@validate_bench +def call_bench_supervisorctl(bench: str): + data = request.json + _bench = Server().benches[bench] + job = _bench.call_supervisorctl( + data["command"], + data["programs"], + ) + return {"job": job} + + +@application.errorhandler(BenchNotExistsException) +def bench_not_found(e): + return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 404 + + +@application.errorhandler(SiteNotExistsException) +def site_not_found(e): + return {"error": "".join(traceback.format_exception(*sys.exc_info())).splitlines()}, 404 + + +@application.route("/docker_cache_utils/", methods=["POST"]) +def docker_cache_utils(method: str): + from agent.docker_cache_utils import ( + get_cached_apps, + run_command_in_docker_cache, + ) + + if method == "run_command_in_docker_cache": + return run_command_in_docker_cache(**request.json) + + if method == "get_cached_apps": + return get_cached_apps() + + return None + + +@application.route("/benches//update_inplace", methods=["POST"]) +def update_inplace(bench: str): + sites = request.json.get("sites") + apps = request.json.get("apps") + image = request.json.get("image") + _bench = Server().benches[bench] + job = _bench.update_inplace( + sites, + image, + apps, + ) + return {"job": job} + + +@application.route("/benches//recover_update_inplace", methods=["POST"]) +def recover_update_inplace(bench: str): + _bench = Server().benches[bench] + job = _bench.recover_update_inplace( + request.json.get("sites"), + request.json.get("image"), + ) + return {"job": job} diff --git a/setup.py b/setup.py index 8835fc7..7349b9e 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,22 @@ -from setuptools import find_packages, setup - -with open("requirements.txt") as f: - install_requires = f.read().strip().split("\n") - - -setup( - name="agent", - version="0.0.0", - description="Jingrow Jcloud Agent", - url="http://git.jingrow.com:3000/jingrow/agent", - author="Jingrow Technologies", - author_email="developers@framework.jingrow.com", - packages=find_packages(), - zip_safe=False, - install_requires=install_requires, - entry_points={ - "console_scripts": [ - "agent = agent.cli:cli", - ], - }, -) +from setuptools import find_packages, setup + +with open("requirements.txt") as f: + install_requires = f.read().strip().split("\n") + + +setup( + name="agent", + version="0.0.0", + description="Jingrow Jcloud Agent", + url="http://git.jingrow.com/jingrow/agent", + author="Jingrow Technologies", + author_email="developers@framework.jingrow.com", + packages=find_packages(), + zip_safe=False, + install_requires=install_requires, + entry_points={ + "console_scripts": [ + "agent = agent.cli:cli", + ], + }, +)