[
 {
  "callback_max_retry": 5,
  "callback_script": "snapshot = jingrow.get_pg(\"Server Snapshot\", arguments.get(\"server_snapshot\"))\nif pg.server_type == \"Server\":\n\tsnapshot.resume_app_server_services()\nelif pg.server_type == \"Database Server\":\n\tsnapshot.resume_database_server_services()\n\nif pg.status == \"Failure\":\n    jingrow.db.set_value(\"Server Snapshot\", snapshot.name, \"status\", \"Failure\", update_modified=False)\n    for s in snapshot.snapshots:\n        try:\n            jingrow.get_pg(\"Virtual Disk Snapshot\", s).delete_snapshot()\n        except:\n            pass\n        \nelse:\n    snapshot.sync(now=False)",
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-09-24 16:33:10.530624",
  "name": "Snapshot Disk",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n\nwhile True:\n    try:\n        machine.sync()\n    finally:\n        break\n\nif machine.status == \"Terminated\":\n    raise Exception(\"Can't snapshot terminated virtual machine\")\nelif machine.status == \"Draft\":\n    raise Exception(\"Can't snapshot draft virtual machine\")",
    "step_name": "Verify Virtual Machine Status",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Server\" and arguments.get(\"is_consistent_snapshot\", False):\n    server = jingrow.get_pg(\"Server\", pg.server)\n    output = server.ansible_run(\"systemctl stop docker\")\n    if not (output and output.get(\"status\") == \"Success\"):\n        raise Exception(\"Failed to stop docker daemon\")\nelse:\n    result = (False, False) # Skipped\n",
    "step_name": "Stop Docker Daemon",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\" and arguments.get(\"is_consistent_snapshot\", False):\n    server = jingrow.get_pg(\"Database Server\", pg.server)\n    output = server.ansible_run(\"systemctl stop mariadb\")\n    if not (output and output.get(\"status\") == \"Success\"):\n        raise Exception(\"Failed to stop mariadb service\")\n    result = (True, False)\nelse:\n    result = (False, False) # Skipped\n",
    "step_name": "Stop MariaDB Service",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\noutput = server.ansible_run(\"sync\")\nif not (output and output.get(\"status\") == \"Success\"):\n    raise Exception(\"Failed to flush file system buffers to disk\")\n",
    "step_name": "Flush File System Buffers to Disk",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.create_snapshots(exclude_boot_volume=True, dedicated_snapshot=True)\n\nfield_name = \"app_server_snapshot\" if pg.server_type == \"Server\" else \"database_server_snapshot\"\nno_of_snapshots = len(machine.flags.created_snapshots)\nif no_of_snapshots != 1:\n    raise Exception(\"Expected 1 disk snapshot. Found : \"+str(no_of_snapshots))\n    \njingrow.db.set_value(\"Server Snapshot\", arguments.get(\"server_snapshot\"), field_name, machine.flags.created_snapshots[0], update_modified=False)",
    "step_name": "Snapshot Disk",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-08-12 16:24:10.555919",
  "name": "Attach Volume",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nprint(machine.as_dict())\nif machine.cloud_provider in [\"AWS EC2\", \"OCI\"]:\n    machine.attach_new_volume(machine.size, machine.iops, machine.throughput)\nelse:\n    machine.attach_volume(size=100)",
    "step_name": "Attach Volume",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-01-02 15:33:27.252601",
  "name": "Resize Server",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.stop()",
    "step_name": "Stop Virtual Machine",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Stopped\", False)",
    "step_name": "Wait for Virtual Machine to Stop",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.resize(arguments.machine_type)",
    "step_name": "Resize Virtual Machine",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\ntry:\n    machine.start()\n    result = (True, False)\nexcept:\n    result = (False, False)",
    "step_name": "Start Virtual Machine",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Running\", False)",
    "step_name": "Wait for Virtual Machine to Start",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.ping_ansible()\n\nplays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Ping Server\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", False)\n",
    "step_name": "Wait for Server to be accessible",
    "wait_until_true": 1
   },
   {
    "script": "if pg.server_type == \"Database Server\":\n    server = jingrow.get_pg(pg.server_type, pg.server)\n    server.adjust_memory_config()\nelif pg.server_type == \"Server\":\n    server = jingrow.get_pg(pg.server_type, pg.server)\n    server.auto_scale_workers()",
    "step_name": "Set additional config",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n\nserver = jingrow.get_pg(pg.server_type, pg.server)\nif server.plan:\n    plan_disk_size = jingrow.db.get_value(\"Server Plan\", server.plan, \"disk\")\n    if plan_disk_size and plan_disk_size > machine.disk_size:\n        try:\n            server.increase_disk_size(increment=plan_disk_size - machine.disk_size)\n        except:\n            pass",
    "step_name": "Increase Disk Size",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2024-02-05 17:08:00.514456",
  "name": "Create Server Snapshot",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.stop()\n",
    "step_name": "Stop Virtual Machine",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Stopped\", False)",
    "step_name": "Wait for Virtual Machine to Stop",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\narguments.image = machine.create_image()",
    "step_name": "Create Snapshot",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\ntry:\n    machine.start()\n    result = (True, False)\nexcept:\n    result = (False, False)",
    "step_name": "Start Virtual Machine",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Running\", False)",
    "step_name": "Wait for Virtual Machine to Start",
    "wait_until_true": 1
   },
   {
    "script": "image = jingrow.get_pg(\"Virtual Machine Image\", arguments.image)\nimage.sync()\nresult = (image.status == \"Available\", False)",
    "step_name": "Wait for Snapshot to Complete",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type in [\"Server\", \"Database Server\"] and server.is_for_recovery:\n    filter_field = \"app_server\" if pg.server_type == \"Server\" else \"database_server\"\n    recovery_record_name = jingrow.db.get_value(\"Server Snapshot Recovery\", {filter_field: pg.server}, \"name\")\n    if recovery_record_name:\n        recovery_record = jingrow.get_pg(\"Server Snapshot Recovery\", recovery_record_name, for_update=True)\n    \n        if pg.status == \"Success\":\n            if pg.server_type == \"Server\":\n                recovery_record.app_server_archived = True\n            else:\n                recovery_record.database_server_archived = True\n            recovery_record.save()",
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-09-08 11:42:40.490054",
  "name": "Archive Server",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.disable_termination_protection()",
    "step_name": "Disable Termination Protection",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.terminate()",
    "step_name": "Terminate Virtual Machine",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\ntry:\n    # Usually machine sync never fails\n    # It can fail weirdly due to TimestampMismatchError or lock timeout\n    # Don't fail this whole thing just because of that\n    # Ignore the errors of sync and keep retrying\n    machine.sync()\nexcept:\n    pass\nresult = (machine.status == \"Terminated\", False)",
    "step_name": "Wait for Virtual Machine to Terminate",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2024-01-05 13:40:21.038901",
  "name": "Upgrade MariaDB",
  "steps": [
   {
    "script": "server = jingrow.get_pg(\"Database Server\", pg.server)\nserver.stop_mariadb()",
    "step_name": "Stop MariaDB",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Stop MariaDB\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", False)\n",
    "step_name": "Wait for MariaDB to Stop",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.create_snapshots()\nsnapshot = jingrow.get_last_pg(\"Virtual Disk Snapshot\", {\"virtual_machine\": pg.virtual_machine})\nsnapshot.add_comment(text=\"Before MariaDB Upgrade\")",
    "step_name": "Create Server Snapshot",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(\"Database Server\", pg.server)\nserver.upgrade_mariadb()",
    "step_name": "Upgrade MariaDB",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Upgrade MariaDB\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\n",
    "step_name": "Wait for MariaDB Upgrade to Complete",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-02-14 14:30:00.676187",
  "name": "Increase Disk Size",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nmountpoint = arguments.labels.get(\"mountpoint\")\nserver.calculated_increase_disk_size(mountpoint=mountpoint)",
    "step_name": "Increase Disk Size",
    "wait_until_true": 0
   },
   {
    "script": "should_auto_increase = jingrow.db.get_value(pg.server_type, pg.server, \"auto_increase_storage\")\nif not should_auto_increase:\n    result = (True, False)\n\nelse:\n    if jingrow.db.get_value(pg.server_type, pg.server, \"provider\") == \"AWS EC2\":\n        plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Extend EC2 Volume\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n        result = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\n    else:\n        result = (True, False)",
    "step_name": "Wait for partition to resize (AWS)",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"OCI\":\n    machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n    machine.sync()\n    result = (machine.status == \"Running\", False)\nelse:\n    result = (True, False)",
    "step_name": "Wait for server to start (OCI)",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"OCI\":\n    server.ping_ansible()\n\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Ping Server\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", False)\nelse:\n    result = (True, False)\n",
    "step_name": "Wait for server to be accessible (OCI)",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"OCI\":\n    server.add_glass_file()\nelse:\n    result = (True, False)\n    # handled for aws already in extend playbook",
    "step_name": "Add glass file back (OCI)",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == 'Server':\n    server = jingrow.get_pg(pg.server_type, pg.server)\n    server.start_active_benches()\n",
    "step_name": "Restart Active Benches",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-11-12 09:35:41.121169",
  "name": "Prune Docker system",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\ntelegram_message = jingrow.get_pg(\"Jcloude Settings\").telegram_message\ntelegram_message.enqueue(f\"Pruning docker cache on [{server.name}]({jingrow.utils.get_url_to_form(server.pagetype, server.name)})\", \"Information\")\nserver.prune_docker_system()",
    "step_name": "Prune Docker system",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Prune Docker System\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\n",
    "step_name": "Wait for docker system prune",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-11-12 09:35:21.819679",
  "name": "Increase Swap",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\ntelegram_message = jingrow.get_pg(\"Jcloude Settings\").telegram_message\ntelegram_message.enqueue(f\"Increasing swap on [{server.name}]({jingrow.utils.get_url_to_form(server.pagetype, server.name)})\", \"Information\")\nserver.increase_swap(4)",
    "step_name": "Add swap on server",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Increase Swap\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")",
    "step_name": "Wait for swap to be added",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2024-12-06 10:59:08.032149",
  "name": "Stop and Start Server",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.stop()",
    "step_name": "Stop Virtual Machine",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Stopped\", False)",
    "step_name": "Wait for Virtual Machine to Stop",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\ntry:\n    machine.start()\n    result = (True, False)\nexcept:\n    result = (False, False)",
    "step_name": "Start Virtual Machine",
    "wait_until_true": 1
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Running\", False)",
    "step_name": "Wait for Virtual Machine to Start",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.ping_ansible()\n\nplays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Ping Server\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", False)\n",
    "step_name": "Wait for Server to be accessible",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-11-12 09:34:59.467479",
  "name": "Reset Swap",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\ntelegram_message = jingrow.get_pg(\"Jcloude Settings\").telegram_message\ntelegram_message.enqueue(f\"Resetting swap on [{server.name}]({jingrow.utils.get_url_to_form(server.pagetype, server.name)})\", \"Information\")\nserver.reset_swap()",
    "step_name": "Reset swap on server",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Reset Swap\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")",
    "step_name": "Wait for swap to be reset",
    "wait_until_true": 1
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-07-11 15:20:56.780290",
  "name": "Warn disk at 80%",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nmountpoint = arguments.labels.get(\"mountpoint\")\nserver.recommend_disk_increase(mountpoint=mountpoint)",
    "step_name": "Send Warning",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-07-25 21:43:11.895128",
  "name": "Trigger Build Server Cleanup",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif server.use_for_build:\n    server.prune_docker_system()\n    \n",
    "step_name": "Trigger Build Server Cleanup",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-08-31 20:54:46.857348",
  "name": "Create Server (old)",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.provision()\n",
    "step_name": "Create Server",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.sync()\nresult = (machine.status == \"Running\", False)\n",
    "step_name": "Wait for Server to start",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.ping_ansible()\n\nplays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Ping Server\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", False)\n",
    "step_name": "Wait for Server to be accessible",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.wait_for_cloud_init()",
    "step_name": "Check Cloud Init status",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Wait for Cloud Init to finish\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for Cloud Init to finish",
    "wait_until_true": 1
   },
   {
    "script": "provider = jingrow.db.get_value(pg.server_type, pg.server, 'provider')\nif provider == \"Hetzner\" and pg.server_type != \"Proxy Server\":\n    vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n    vm.attach_new_volume(100)\n    vm.sync()\n    server = jingrow.get_pg(pg.server_type, pg.server)\n    server.validate_mounts()\n    server.save()\n    server.mount_volumes()\n",
    "step_name": "Create and mount volumes",
    "wait_until_true": 0
   },
   {
    "script": "provider = jingrow.db.get_value(pg.server_type, pg.server, 'provider')\nif provider == 'Hetzner' and pg.server_type != \"Proxy Server\":\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Mount Volumes\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", False)\nelse:\n    result = (True, False)",
    "step_name": "Wait for volumes to mount",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"Hetzner\" :\n    if server.pagetype == \"Server\":\n        server.setup_docker()\n    elif server.pagetype == \"Database Server\":\n        server.set_mariadb_mount_dependency()\n        ",
    "step_name": "Configure apps for mounts",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.update_tls_certificate()",
    "step_name": "Update TLS Certificate",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Setup TLS Certificates\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for TLS Certificate to be updated",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.update_agent_ansible()",
    "step_name": "Update Agent Ansible",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Update Agent\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for Agent to be updated",
    "wait_until_true": 1
   },
   {
    "script": "if pg.server_type == \"Database Server\":\n    server = jingrow.get_pg(\"Database Server\", pg.server)\n    server.upgrade_mariadb()",
    "step_name": "Upgrade MariaDB",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\":\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Upgrade MariaDB\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\nelse:\n    result = (True,)",
    "step_name": "Wait for MariaDB Upgrade to Complete",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.set_additional_config()",
    "step_name": "Set additional config",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 0,
  "callback_script": "",
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-09-08 11:36:55.450275",
  "name": "Resume Services After Snapshot",
  "steps": [
   {
    "script": "if pg.server_type == \"Server\" and arguments.get(\"is_consistent_snapshot\", False):\n    server = jingrow.get_pg(\"Server\", pg.server)\n    output = server.ansible_run(\"systemctl start docker\")\n    if not (output and output.get(\"status\") == \"Success\"):\n        raise Exception(\"Failed to start docker daemon\")\n\n    jingrow.db.set_value(\"Server Snapshot\", arguments.get(\"server_snapshot\"), \"app_server_services_started\", True, update_modified=False)\nelse:\n    result = (False, False) # Skipped\n    # As it wasn't a consistent snapshot services were never stopped\n    jingrow.db.set_value(\"Server Snapshot\", arguments.get(\"server_snapshot\"), \"app_server_services_started\", True, update_modified=False)",
    "step_name": "Start Docker Daemon",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\" and arguments.get(\"is_consistent_snapshot\", False):\n    server = jingrow.get_pg(\"Database Server\", pg.server)\n    output = server.ansible_run(\"systemctl start mariadb\")\n    if not (output and output.get(\"status\") == \"Success\"):\n        raise Exception(\"Failed to start mariadb service\")\n    jingrow.db.set_value(\"Server Snapshot\", arguments.get(\"server_snapshot\"), \"database_server_services_started\", True, update_modified=False)\n\nelse:\n    result = (False, False) # Skipped\n    # As it wasn't a consistent snapshot services were never stopped\n    jingrow.db.set_value(\"Server Snapshot\", arguments.get(\"server_snapshot\"), \"database_server_services_started\", True, update_modified=False)",
    "step_name": "Start MariaDB Service",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 1,
  "callback_script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type in [\"Server\", \"Database Server\"] and server.is_for_recovery:\n    filter_field = \"app_server\" if pg.server_type == \"Server\" else \"database_server\"\n    recovery_record_name = jingrow.db.get_value(\"Server Snapshot Recovery\", {filter_field: pg.server}, \"name\")\n    if recovery_record_name:\n        recovery_record = jingrow.get_pg(\"Server Snapshot Recovery\", recovery_record_name, for_update=True)\n    \n        if pg.status == \"Success\":\n            if pg.server_type == \"Server\":\n                recovery_record.is_app_server_ready = True\n            else:\n                recovery_record.is_database_server_ready = True\n            recovery_record.save()\n        else:\n            recovery_record.mark_server_provisioning_as_failed()\n            \nif pg.server_type in [\"Server\", \"Database Server\"] and \"logical_replication_backup\" in arguments:\n    if pg.status == \"Success\":\n        jingrow.get_pg(\"Logical Replication Backup\", arguments.get(\"logical_replication_backup\")).next()\n    if pg.status == \"Failure\":\n        jingrow.get_pg(\"Logical Replication Backup\", arguments.get(\"logical_replication_backup\")).fail()",
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-10-18 20:59:59.413236",
  "name": "Create Server",
  "steps": [
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nmachine.provision()\n",
    "step_name": "Create Server",
    "wait_until_true": 0
   },
   {
    "script": "machine = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\ntry:\n    # Usually machine sync never fails\n    # It can fail weirdly due to TimestampMismatchError or lock timeout\n    # Don't fail this whole thing just because of that\n    # Ignore the errors of sync and keep retrying\n    machine.sync()\n\nexcept (jingrow.QueryDeadlockError, jingrow.QueryTimeoutError, jingrow.TimestampMismatchError):\n    result = (False, False)\nelse:\n    result = (machine.status == \"Running\", False)\n",
    "step_name": "Wait for Server to start",
    "wait_until_true": 1
   },
   {
    "script": "provider = jingrow.db.get_value(pg.server_type, pg.server, 'provider')\nif provider == \"Hetzner\":\n    vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n    vm.correct_private_ip() #blocking\n    vm.enable_termination_protection() #blocking",
    "step_name": "Correct private IP (Hetzner)",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.ping_ansible()\n\nplays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Ping Server\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status == \"Success\", False)\n",
    "step_name": "Wait for Server to be accessible",
    "wait_until_true": 1
   },
   {
    "script": "try:\n    vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n    vm.sync()\n    if len(vm.volumes) > 0:\n        result = (True, False)\n    else:\n        result = (False, False)\nexcept (jingrow.QueryDeadlockError, jingrow.QueryTimeoutError, jingrow.TimestampMismatchError):\n    result = (False, False)\nexcept Exception as e:\n    raise e",
    "step_name": "Sync Default Volumes",
    "wait_until_true": 1
   },
   {
    "script": "vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nif vm.data_disk_snapshot:\n    is_created = vm.create_data_disk_volume_from_snapshot()\n    if is_created:\n        result = (True, False)\n    else:\n        arguments.update({\"max_volume_creation_retries\": arguments.get(\"max_volume_creation_retries\", 6)-1})\n        if arguments.get(\"max_volume_creation_retries\") <= 0:\n            result = (False, True)\n        result = (False, False)",
    "step_name": "Create Volume From Snapshot",
    "wait_until_true": 1
   },
   {
    "script": "vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\nif vm.data_disk_snapshot:\n    is_attached = vm.check_and_attach_data_disk_snapshot_volume()\n    if is_attached:\n        result = (True, False)\n    else:\n        result = (False, False)",
    "step_name": "Attach Snapshotted Volume",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"AWS EC2\" and jingrow.db.get_value(\"Virtual Machine\", server.virtual_machine, \"data_disk_snapshot\"):\n    try:\n        vm = jingrow.get_pg(\"Virtual Machine\", server.virtual_machine)\n        vm.sync()\n        \n        if len(vm.volumes) == 0 or (vm.data_disk_snapshot_attached and len(vm.volumes) == 1):\n            result = (False, False)\n        else:\n            server.reload()\n            server.validate_mounts()\n            server.save()\n            result = (True, False)\n    except (jingrow.QueryDeadlockError, jingrow.QueryTimeoutError, jingrow.exceptions.TimestampMismatchError):\n        result = (False, False)\n    except Exception as e:\n        raise e\nelse:\n    result = (True, False)",
    "step_name": "Sync Attached Volumes",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"AWS EC2\" and jingrow.db.get_value(\"Virtual Machine\", server.virtual_machine, \"data_disk_snapshot\"):\n    cleanup_db_replication_files = False\n    if pg.server_type == \"Database Server\" and (server.is_for_recovery or arguments.get(\"setup_db_replication\", False)):\n        cleanup_db_replication_files = True\n    server.mount_volumes(\n        now=False,\n        stop_docker_before_mount=pg.server_type == \"Server\",\n        stop_mariadb_before_mount=pg.server_type == \"Database Server\",\n        # If server is in recovery mode, don't start docker and containers\n        # Because If site gets active, background job witll be started and that can modify data\n        start_docker_after_mount=pg.server_type == \"Server\" and not server.is_for_recovery,\n        # If goal is to create replica server, don't start database\n        # As we need to do some additional config before starting database\n        start_mariadb_after_mount=pg.server_type == \"Database Server\" and not arguments.get(\"setup_db_replication\", False),\n        cleanup_db_replication_files=cleanup_db_replication_files,\n        # It's important to change uuid, labels of attached disk\n        # There is high chance that the root disk and data disk might have same disk info\n        rotate_additional_volume_metadata=True\n    )\nelse:\n    result = (False, False)",
    "step_name": "Mount Data Disk",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"AWS EC2\" and jingrow.db.get_value(\"Virtual Machine\", server.virtual_machine, \"data_disk_snapshot\"):\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Mount Volumes\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\nelse:\n    result = None",
    "step_name": "Wait for Data Disk Mount to Complete",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.wait_for_cloud_init()",
    "step_name": "Check Cloud Init status",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Wait for Cloud Init to finish\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for Cloud Init to finish",
    "wait_until_true": 1
   },
   {
    "script": "provider = jingrow.db.get_value(pg.server_type, pg.server, 'provider')\nif provider == \"Hetzner\" and pg.server_type != \"Proxy Server\":\n    vm = jingrow.get_pg(\"Virtual Machine\", pg.virtual_machine)\n    vm.attach_new_volume(100)\n    vm.sync()\n    server = jingrow.get_pg(pg.server_type, pg.server)\n    server.validate_mounts()\n    server.save()\n    server.mount_volumes(now=False)\n",
    "step_name": "Create and mount volumes (Hetzner)",
    "wait_until_true": 0
   },
   {
    "script": "provider = jingrow.db.get_value(pg.server_type, pg.server, 'provider')\nif provider == 'Hetzner' and pg.server_type != \"Proxy Server\":\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Mount Volumes\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", False)\nelse:\n    result = (True, False)",
    "step_name": "Wait for volumes to mount",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nif server.provider == \"Hetzner\" :\n    if server.pagetype == \"Server\":\n        server.setup_docker()\n    elif server.pagetype == \"Database Server\":\n        server.set_mariadb_mount_dependency()",
    "step_name": "Configure apps for mounts (Hetzner)",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.update_tls_certificate()",
    "step_name": "Update TLS Certificate",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Setup TLS Certificates\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for TLS Certificate to be updated",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.update_agent_ansible()",
    "step_name": "Update Agent Ansible",
    "wait_until_true": 0
   },
   {
    "script": "plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Update Agent\"}, [\"status\"], order_by=\"creation desc\", limit=1)\nresult = (plays and plays[0].status in (\"Success\", \"Failure\"), False)",
    "step_name": "Wait for Agent to be updated",
    "wait_until_true": 1
   },
   {
    "script": "if pg.server_type == \"Database Server\" and arguments.get(\"setup_db_replication\", False):\n    pass",
    "step_name": "Stop MariaDB Slave",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\":\n    server = jingrow.get_pg(\"Database Server\", pg.server)\n    server.upgrade_mariadb()",
    "step_name": "Upgrade MariaDB",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\":\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Upgrade MariaDB\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\nelse:\n    result = (True,)",
    "step_name": "Wait for MariaDB Upgrade to Complete",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Database Server\" and arguments.get(\"setup_db_replication\", False):\n    server.prepare_mariadb_replica()\n    result = (True, False)\nelse:\n    result = (False, False)",
    "step_name": "Prepare MariaDB Replica",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server_type == \"Database Server\" and arguments.get(\"setup_db_replication\", False):\n    plays = jingrow.get_all(\"Ansible Play\", {\"server\": pg.server, \"play\": \"Prepare MariaDB Replica\"}, [\"status\"], order_by=\"creation desc\", limit=1)\n    result = (plays and plays[0].status == \"Success\", plays and plays[0].status == \"Failure\")\nelse:\n    result = (True,)",
    "step_name": "Wait for MariaDB Replica to Be Prepared",
    "wait_until_true": 1
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Database Server\" and arguments.get(\"setup_db_replication\", False):\n    server.configure_replication()\n    result = (True, False)\nelse:\n    result = (False, False)",
    "step_name": "Configure MariaDB Replica",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Database Server\" and arguments.get(\"setup_db_replication\", False):\n    server.start_replication()\n    result = (True, False)\nelse:\n    result = (False, False)",
    "step_name": "Start MariaDB Replica",
    "wait_until_true": 0
   },
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\nserver.set_additional_config()",
    "step_name": "Set additional config",
    "wait_until_true": 0
   },
   {
    "script": "if pg.server.startswith(\"fs\") and pg.server_type == \"Server\":\n    primary_server = jingrow.db.get_value(\"Server\", pg.server, \"primary\")\n    nfs_volume_attachment = jingrow.get_pg(\n\t    {\"pagetype\": \"NFS Volume Attachment\", \"primary_server\": primary_server}\n    )\n    nfs_volume_attachment.insert(ignore_permissions=True)\n    jingrow.db.commit()",
    "step_name": "Share benches over NFS",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 1,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-11-12 11:06:08.243873",
  "name": "Prune Mirror Registry",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Registry Server\":\n    server.prune_mirror_registry()",
    "step_name": "Prune Mirror Registry",
    "wait_until_true": 0
   }
  ]
 },
 {
  "callback_max_retry": 1,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-12-18 15:28:08.243873",
  "name": "Auto Scale Up Application Server",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Server\" and not server.scaled_up:\n    server.scale_up(is_automatically_triggered=True)",
    "step_name": "Auto Scale Up Application Server",
    "wait_until_true": 0
    }
   ]
 },
 {
  "callback_max_retry": 1,
  "callback_script": null,
  "pagestatus": 0,
  "pagetype": "Jcloude Job Type",
  "modified": "2025-12-18 15:28:08.243873",
  "name": "Auto Scale Down Application Server",
  "steps": [
   {
    "script": "server = jingrow.get_pg(pg.server_type, pg.server)\n\nif pg.server_type == \"Server\" and server.scaled_up:\n    server.scale_down(is_automatically_triggered=True)",
    "step_name": "Auto Scale Down Application Server",
    "wait_until_true": 0
    }
   ]
 }  
]
