Browse Source

chore: add restic periodic job

pull/2/head
Karan Sharma 8 months ago
parent
commit
befff8427e
  1. 2
      SETUP.md
  2. 4
      terraform/env.sample
  3. 19
      terraform/main.tf
  4. 9
      terraform/modules/caddy/conf/Caddyfile-public
  5. 26
      terraform/modules/cloudflare/records.tf
  6. 122
      terraform/modules/gitea/conf/app.ini.tpl
  7. 86
      terraform/modules/gitea/conf/gitea.nomad
  8. 9
      terraform/modules/gitea/data.tf
  9. 8
      terraform/modules/gitea/job.tf
  10. 4
      terraform/modules/gitea/variables.tf
  11. 17
      terraform/modules/restic/conf/backup.sh
  12. 41
      terraform/modules/restic/conf/restic.nomad
  13. 7
      terraform/modules/restic/job.tf
  14. 22
      terraform/variables.tf

2
setup.md → SETUP.md

@ -1,5 +1,7 @@
# Nomad Cluster Setup
P
# Setup Tailscale
Tailscale acts as a mesh layer between the server and worker nodes. Since the user's laptop/mobile also has a Tailscale agent running it makes it easy to deploy and browse Nomad/Consul Admin UIs as well.

4
terraform/env.sample

@ -4,3 +4,7 @@ TF_VAR_cloudflare_caddy_api_token=
TF_VAR_shynet_postgresql_password=
TF_VAR_shynet_django_secret_key=
TF_VAR_joplin_postgresql_password=
TF_VAR_gitea_secret_key=
TF_VAR_gitea_internal_token=
TF_VAR_gitea_lfs_jwt_secret=
TF_VAR_gitea_oauth2_jwt_secret=

19
terraform/main.tf

@ -44,3 +44,22 @@ module "joplin" {
nomad = nomad
}
}
module "restic" {
source = "./modules/restic"
providers = {
nomad = nomad
}
}
module "gitea" {
source = "./modules/gitea"
gitea_secret_key = var.gitea_secret_key
gitea_internal_token = var.gitea_internal_token
gitea_lfs_jwt_secret = var.gitea_lfs_jwt_secret
gitea_oauth2_jwt_secret = var.gitea_oauth2_jwt_secret
providers = {
nomad = nomad
}
}

9
terraform/modules/caddy/conf/Caddyfile-public

@ -7,3 +7,12 @@ shynet.mrkaran.dev {
}
}
git.mrkaran.dev {
{{ range service "gitea-web" }}
reverse_proxy {{ .Address }}:{{ .Port }}
{{ end }}
tls {
dns cloudflare "${cloudflare_api_token}"
}
}

26
terraform/modules/cloudflare/records.tf

@ -66,7 +66,7 @@ resource "cloudflare_record" "shynet" {
ttl = "1"
proxied = "true"
# TODO: shift to floating IP
value = "68.183.87.4"
value = var.ips["floyd_public"]
}
@ -77,6 +77,30 @@ resource "cloudflare_record" "joplin" {
type = "A"
ttl = "1"
proxied = "false"
value = var.ips["floyd_tailscale"]
}
resource "cloudflare_record" "koadings" {
zone_id = cloudflare_zone.mrkaran_dev.id
name = "koadings"
type = "A"
ttl = "1"
proxied = "false"
value = var.ips["floyd_tailscale"]
}
resource "cloudflare_record" "git" {
zone_id = cloudflare_zone.mrkaran_dev.id
name = "git"
type = "A"
ttl = "1"
proxied = "true"
value = var.ips["floyd_public"]
}

122
terraform/modules/gitea/conf/app.ini.tpl

@ -0,0 +1,122 @@
; This file lists the default values used by Gitea
; Sample file: https://github.com/go-gitea/gitea/blob/master/custom/conf/app.example.ini
; Docs: https://docs.gitea.io/en-us/config-cheat-sheet/
APP_NAME = Code by mrkaran
RUN_MODE = prod
RUN_USER = git
[repository]
ROOT = /data/git/repositories
[repository.local]
LOCAL_COPY_PATH = /data/gitea/tmp/local-repo
[repository.upload]
TEMP_PATH = /data/gitea/uploads
[server]
APP_DATA_PATH = /data/gitea
DOMAIN = git.mrkaran.dev
SSH_DOMAIN = koadings.mrkaran.dev
HTTP_PORT = 3000
ROOT_URL = https://git.mrkaran.dev/
DISABLE_SSH = false
SSH_PORT = 4222
SSH_LISTEN_PORT = 22
LFS_START_SERVER = true
LFS_CONTENT_PATH = /data/git/lfs
LFS_JWT_SECRET = "${gitea_lfs_jwt_secret}"
OFFLINE_MODE = true
LANDING_PAGE = explore
[ui]
THEME_COLOR_META_TAG = "#6B46C1"
[ui.meta]
AUTHOR = Karan
DESCRIPTION = Karan's self-hosted Gitea instance
KEYWORDS = git, gitea, karan, git.mrkaran.dev, mrkaran
[database]
PATH = /data/gitea/gitea.db
DB_TYPE = sqlite3
HOST = localhost:3306
NAME = gitea
USER = root
PASSWD =
SCHEMA =
SSL_MODE = disable
CHARSET = utf8
[indexer]
ISSUE_INDEXER_PATH = /data/gitea/indexers/issues.bleve
[session]
PROVIDER_CONFIG = /data/gitea/sessions
PROVIDER = file
COOKIE_SECURE = true
[picture]
AVATAR_UPLOAD_PATH = /data/gitea/avatars
REPOSITORY_AVATAR_UPLOAD_PATH = /data/gitea/repo-avatars
DISABLE_GRAVATAR = false
ENABLE_FEDERATED_AVATAR = true
[attachment]
PATH = /data/gitea/attachments
[log]
ROOT_PATH = /data/gitea/log
MODE = file
LEVEL = info
[security]
INSTALL_LOCK = true
SECRET_KEY = "${gitea_secret_key}"
INTERNAL_TOKEN = "${gitea_internal_token}"
[service]
DISABLE_REGISTRATION = true
REQUIRE_SIGNIN_VIEW = false
REGISTER_EMAIL_CONFIRM = false
ENABLE_NOTIFY_MAIL = false
ALLOW_ONLY_EXTERNAL_REGISTRATION = false
ENABLE_CAPTCHA = false
DEFAULT_KEEP_EMAIL_PRIVATE = false
DEFAULT_ALLOW_CREATE_ORGANIZATION = true
DEFAULT_ENABLE_TIMETRACKING = true
NO_REPLY_ADDRESS = noreply.localhost
[oauth2]
JWT_SECRET = "${gitea_oauth2_jwt_secret}"
[mailer]
ENABLED = false
[openid]
ENABLE_OPENID_SIGNIN = true
ENABLE_OPENID_SIGNUP = true
[metrics]
ENABLED = true
[cron]
ENABLED = true
RUN_AT_START = false
[cron.archive_cleanup]
RUN_AT_START = true
SCHEDULE = @every 24h
OLDER_THAN = 24h
[cron.update_mirrors]
SCHEDULE = @every 3h
[cron.repo_health_check]
SCHEDULE = @every 24h
TIMEOUT = 60s
[cron.check_repo_stats]
RUN_AT_START = true
SCHEDULE = @every 24h

86
terraform/modules/gitea/conf/gitea.nomad

@ -0,0 +1,86 @@
job "gitea" {
datacenters = ["hydra"]
type = "service"
group "app" {
count = 1
network {
port "http" {
to = 3000
}
port "ssh" {
to = 22
# Need a static assignment for SSH ops.
static = 4222
# SSH port on host only exposed to Tailscale IP.
host_network = "tailscale"
}
}
restart {
attempts = 2
interval = "2m"
delay = "30s"
mode = "fail"
}
task "web" {
driver = "docker"
service {
name = "gitea-web"
tags = ["gitea", "web"]
port = "http"
}
config {
image = "gitea/gitea:latest"
ports = ["http"]
# Bind the config file to container.
mount {
type = "bind"
source = "local/gitea.ini"
target = "/data/gitea/conf/app.ini"
}
# Bind the data directory to preserve data.
mount {
type = "bind"
target = "/data"
source = "/data/gitea/"
readonly = false
}
}
template {
data = <<EOF
${gitea_config}
EOF
destination = "local/gitea.ini" # Rendered template.
change_mode = "restart"
# HACK:
# https://github.com/hashicorp/nomad/issues/5020#issuecomment-778608068
perms = "777"
}
env {
# owner of `/data/gitea` on host.
USER_UID = 1000
USER_GID = 1000
}
resources {
cpu = 200
memory = 300
}
}
}
}

9
terraform/modules/gitea/data.tf

@ -0,0 +1,9 @@
data "template_file" "gitea-config" {
template = file("${path.module}/conf/app.ini.tpl")
vars = {
gitea_secret_key = var.gitea_secret_key
gitea_internal_token = var.gitea_internal_token
gitea_lfs_jwt_secret = var.gitea_lfs_jwt_secret
gitea_oauth2_jwt_secret = var.gitea_oauth2_jwt_secret
}
}

8
terraform/modules/gitea/job.tf

@ -0,0 +1,8 @@
resource "nomad_job" "app" {
jobspec = templatefile("${path.module}/conf/gitea.nomad", {
gitea_config = data.template_file.gitea-config.rendered
})
hcl2 {
enabled = true
}
}

4
terraform/modules/gitea/variables.tf

@ -0,0 +1,4 @@
variable "gitea_secret_key" {}
variable "gitea_internal_token" {}
variable "gitea_lfs_jwt_secret" {}
variable "gitea_oauth2_jwt_secret" {}

17
terraform/modules/restic/conf/backup.sh

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set -o allexport; source restic.env; set +o allexport
echo "Starting backup!"
restic backup --verbose --one-file-system --tag nomad /data
echo "Pruning old backups according to retention policy."
restic forget --verbose --tag nomad --group-by "paths,tags" --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 3
echo "Pruning old backups from local storage."
restic prune

41
terraform/modules/restic/conf/restic.nomad

@ -0,0 +1,41 @@
job "restic" {
datacenters = ["hydra"]
type = "batch"
periodic {
cron = "0 * * * *"
time_zone = "Asia/Kolkata"
prohibit_overlap = true
}
group "script" {
count = 1
restart {
attempts = 2
interval = "30m"
delay = "1m"
mode = "fail"
}
task "backup" {
driver = "exec"
config {
# Since `/data` is owned by `root`, restic needs to be spawned as `root`.
command = "sudo"
args = ["./home/karan/restic-backup/backup.sh"]
}
artifact {
source = "https://internal.file.server/name-of-my-binary"
destination = "local/some-directory"
}
resources {
cpu = 200
memory = 50
}
}
}
}

7
terraform/modules/restic/job.tf

@ -0,0 +1,7 @@
resource "nomad_job" "app" {
jobspec = "${path.module}/conf/restic.nomad"
hcl2 {
enabled = true
}
}

22
terraform/variables.tf

@ -3,6 +3,7 @@ locals {
ips = {
floyd_floating = module.servers.floating_floyd
floyd_tailscale = "100.119.138.27"
floyd_public = "68.183.87.4"
}
}
@ -25,3 +26,24 @@ variable "joplin_postgresql_password" {
type = string
description = "Password for joplin PostgreSQL server"
}
variable "gitea_secret_key" {
type = string
description = "Global secret key"
}
variable "gitea_internal_token" {
type = string
description = "Secret used to validate communication within Gitea binary"
}
variable "gitea_lfs_jwt_secret" {
type = string
description = "LFS authentication secret"
}
variable "gitea_oauth2_jwt_secret" {
type = string
description = "OAuth2 authentication secret for access and refresh tokens"
}
Loading…
Cancel
Save