From 5c9107b67fff5d03ca8ad05fd255bc369b1db4d8 Mon Sep 17 00:00:00 2001 From: sn0w Date: Mon, 24 Dec 2018 23:20:17 +0100 Subject: [PATCH 01/21] Start progress on new config --- .env.dist | 70 +------------------------ Dockerfile | 1 + config.dist.yml | 111 ++++++++++++++++++++++++++++++++++++++++ config_parser/parser.rb | 44 ++++++++++++++++ docker-compose.m4 | 27 ++-------- docker-config.exs | 71 ------------------------- entrypoint.sh | 13 ++++- 7 files changed, 174 insertions(+), 163 deletions(-) create mode 100644 config.dist.yml create mode 100755 config_parser/parser.rb delete mode 100644 docker-config.exs diff --git a/.env.dist b/.env.dist index 51013f6..113a0e8 100644 --- a/.env.dist +++ b/.env.dist @@ -1,18 +1,3 @@ -# -# Note: The values of this file that are passed directly to -# pleroma need type hints to be applied correctly. -# The implemented types are int, bool, and string. -# Typed variables look like this: KEY=type:VALUE. -# Not specifying a type implies a string. -# -# In general: Your instance should work as expected if you leave the types as they are. -# That is: don't remove them, don't add them, don't change them. -# Always just change the values. -# -# You don't need to escape colons in your strings. -# PLEROMA_NAME=string:std::pleroma::coolinstance will work as expected. -# - ######################### # Script settings # ######################### @@ -62,15 +47,12 @@ DOCKER_GID=1000 # Database settings # ########################### -# Leave POSTGRES_IP empty unless you plan to install your own database -# Leave the POSTGRES_DB, POSTGRES_USER and POSTGRES_PASSWORD as-is -# unless you use your own database. +# When you use the managed postgres container +# those will be the credentials the container is generated with. -POSTGRES_IP= POSTGRES_DB=pleroma POSTGRES_USER=pleroma POSTGRES_PASSWORD=pleroma -PLEROMA_DB_POOL_SIZE=int:16 ########################## # Pleroma Settings # @@ -81,51 +63,3 @@ MIX_ENV=prod # The git tag, revision, or branch to check out on build PLEROMA_VERSION=develop - -# The loglevel to use -# (error/warn/info/debug) -PLEROMA_LOGLEVEL=error - -# The domain/scheme where pleroma will be hosted -# URL is a bare TLD -# SCHEME is the protocol without "://" -# PORT is the *external* port (ie that of your reverse proxy) -PLEROMA_URL=coolsite.moe -PLEROMA_SCHEME=https -PLEROMA_PORT=int:443 - -# The seed for your secret keys -# (Enter something as random as possible) -# (On linux you can try something like "dd if=/dev/urandom bs=1 count=64 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev") -PLEROMA_SECRET_KEY_BASE= - -# The name of your instance -# (This is displayed in the top-left in pleroma-fe) -PLEROMA_NAME=string:coolInstance - -# Your contact info -PLEROMA_ADMIN_EMAIL=admin@coolsite.moe - -# How many chars a notice may have at max. -PLEROMA_MAX_NOTICE_CHARS=int:500 - -# Whether your instance accepts new users or not (true/false) -PLEROMA_REGISTRATIONS_OPEN=bool:true - -# Enable media proxy (true/false)? -PLEROMA_MEDIA_PROXY_ENABLED=bool:false - -# The url of your media proxy (if enabled) [with "http(s)://"] -PLEROMA_MEDIA_PROXY_URL=string:https://cdn.coolsite.moe - -# Redirect to source on cache fail? -PLEROMA_MEDIA_PROXY_REDIRECT_ON_FAILURE=bool:true - -# Whether to enable the chat feature or not -PLEROMA_CHAT_ENABLED=bool:true - -# Where to store uploads. -# This is only relevant inside the container. -# The host path is always $DOCKER_DATADIR/uploads. -# So, you probably don't need to change this. -PLEROMA_UPLOADS_PATH=/uploads diff --git a/Dockerfile b/Dockerfile index cbaabbd..5050680 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN \ && dpkg -i erlang-solutions_1.0_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends esl-erlang elixir \ + && apt-get install -y ruby \ \ && rm -rf /var/lib/apt/lists/* diff --git a/config.dist.yml b/config.dist.yml new file mode 100644 index 0000000..14d94a9 --- /dev/null +++ b/config.dist.yml @@ -0,0 +1,111 @@ +version: 1 + +# +# Pleroma settings +# +# You can enter any config in here that you want. +# Pleroma-Docker will try to translate it into elixir for you. +# +# For example: +# +# :foo: +# Bar.Baz: +# x: true +# +# becomes `config :foo, Bar.Baz, x: true`. +# +# It is assumed that all config keys that have to be passed through to +# pleroma must start with an atom on the first layer (eg :pleroma). +# + +app: + # The loglevel to use in pleroma. + :logger: + level: info + + :pleroma: + Pleroma.Repo: + # Credentials for your database. + # You should leave this as-is if you want to use the managed db container. + hostname: db + username: pleroma + password: pleroma + database: pleroma + pool_size: 16 + + Pleroma.Web.Endpoint: + # Location where your instance will be reachable. + url: + : Array + scheme: https + host: coolsite.moe + port: 443 + + # Base for your secret keys. + # Better make this random. + secret_key_base: asdf0815 + + Pleroma.Upload: + # Where to store your uploads. + # You should probably leave this as-is. + # /uploads will be mounted into ::docker::datadir. + uploads: /uploads + + # Remove metadata from uploads? + strip_exif: true + + :chat: + # Enable chat functionality? + enabled: true + + :media_proxy: + # Enable the proxy? + enabled: true + + # Emit a 302 to the original resource when uncached? + redirect_on_failure: true + + # Where your proxy is reachable + base_url: https://media.coolsite.moe + + :instance: + # The name of your instance. + name: super cool stuff club + + # Short description of your instance + description: we do super cool stuff in super cool stuff club + + # The admin's email address. + email: admin@coolsite.moe + + # How many chars a notice may have at max. + limit: 4096 + + # May new members sign up? + registrations_open: true + + # Allow connections to other instances? + # (Turn this off for testing) + federating: true + + # The rewrite policies / quarantines to enable. + # This is a powerful feature which should be used with care. + # Take a look at https://git.pleroma.social/pleroma/pleroma/wikis/Message%20rewrite%20facility%20configuration%20(how%20to%20block%20instances). + # Then uncomment only the things you really need + + # quarantined_instances: + # - badguys.moe + + # rewrite_policy: + # - Pleroma.Web.ActivityPub.MRF.SimplePolicy + + # :mrf_simple: + # media_removal: + # - illegalporn.biz + # media_nsfw: + # - porn.biz + # - porn.business + # reject: + # - spam.com + # federated_timeline_removal: + # - spam.university diff --git a/config_parser/parser.rb b/config_parser/parser.rb new file mode 100755 index 0000000..96f1972 --- /dev/null +++ b/config_parser/parser.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby + +require 'yaml' +require 'json' + +config = YAML.load_file(ARGV[0]) + +if config["version"] != 1 + raise "Incompatible config version (#{config["version"]} != 1)" +end + +buf = "use Mix.Config\n\n" + +config["app"].each do |atom, content| + content.each do |sub, settings| + buf += "config :#{atom}, #{sub.is_a?(Symbol) ? ":#{sub}" : sub}" + + if !settings.is_a? Hash + buf += ": #{settings.to_json}\n" + next + end + + settings.each do |name, value| + if value.is_a?(Hash) && value[""] == "Array" + value.delete("") + + buf += ", #{name}: [" + + value.each do |k, v| + buf += "#{k}: #{v.to_json}," + end + buf.chop! + + buf += "]" + else + buf += ", #{name}: #{value.to_json}" + end + end + + buf += "\n" + end +end + +puts buf diff --git a/docker-compose.m4 b/docker-compose.m4 index 1a1b155..d3c445d 100644 --- a/docker-compose.m4 +++ b/docker-compose.m4 @@ -88,7 +88,7 @@ define(, <${upcase($1):-$2}>) "env()", "env()", "env()", - "env_fb(, )", + "env_fb(, )" ] }, "restart": "unless-stopped", @@ -96,31 +96,12 @@ define(, <${upcase($1):-$2}>) ifelse(__SCRIPT_DEPLOY_POSTGRES, true, <"db">) ], "environment": [ - "env_fb(, )", - - "env_fb(, )", - "env()", - "env()", - "env()", - - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env()", - "env_fb(, )" + "env_fb(, )" ], "volumes": [ "./custom.d:/custom.d", + "./config.yml:/conf/config.yml:ro", + "./config_parser/parser.rb:/conf/parser.rb:ro", "env_inline()/uploads:env_inline_fb(, )" ], "labels": [ diff --git a/docker-config.exs b/docker-config.exs deleted file mode 100644 index 5207e38..0000000 --- a/docker-config.exs +++ /dev/null @@ -1,71 +0,0 @@ -use Mix.Config - -defmodule Docker do - def env(shortname, verbatim \\ false) do - # Get var - name = ((if verbatim, do: "", else: "pleroma_") <> Atom.to_string(shortname)) |> String.upcase() - raw_var = System.get_env(name) - - if raw_var == nil do - raise "Could not find #{name} in environment. Please define it and try again." - end - - # Match type and cast if needed - if String.contains?(raw_var, ":") do - var_parts = String.split(raw_var, ":", parts: 2) - - type = Enum.at(var_parts, 0) - var = Enum.at(var_parts, 1) - - func = case type do - "int" -> fn(x) -> Integer.parse(x) |> elem(0) end - "bool" -> fn(x) -> x == "true" end - "string" -> fn(x) -> x end - _ -> if verbatim do - fn(x) -> x end - else - raise "Unknown type #{type} used in variable #{raw_var}." - end - end - - func.(var) - else - raw_var - end - end -end - -config :logger, level: String.to_atom(Docker.env(:loglevel) || "info") - -config :pleroma, Pleroma.Web.Endpoint, - url: [ - host: Docker.env(:url), - scheme: Docker.env(:scheme), - port: Docker.env(:port) - ], - secret_key_base: Docker.env(:secret_key_base) - -config :pleroma, Pleroma.Upload, - uploads: Docker.env(:uploads_path) - -config :pleroma, :chat, - enabled: Docker.env(:chat_enabled) - -config :pleroma, :instance, - name: Docker.env(:name), - email: Docker.env(:admin_email), - limit: Docker.env(:max_notice_chars), - registrations_open: Docker.env(:registrations_open) - -config :pleroma, :media_proxy, - enabled: Docker.env(:media_proxy_enabled), - redirect_on_failure: Docker.env(:media_proxy_redirect_on_failure), - base_url: Docker.env(:media_proxy_url) - -config :pleroma, Pleroma.Repo, - adapter: Ecto.Adapters.Postgres, - username: Docker.env(:postgres_user, true), - password: Docker.env(:postgres_password, true), - database: Docker.env(:postgres_db, true), - hostname: Docker.env(:postgres_ip, true), - pool_size: Docker.env(:db_pool_size) diff --git a/entrypoint.sh b/entrypoint.sh index 35f5365..be5fc93 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,7 +3,18 @@ set -e set -x -mix deps.get +# Generate a config file +ruby /config/parser.rb /config/config.yml > runtime-config.exs + +# Recompile if needed +if [[ ! -z "$RECOMPILE" ]]; then + mix deps.get + mix compile +fi + +# Migrate db mix ecto.create mix ecto.migrate + +# Off we go! exec mix phx.server From bdf83f7227a28d5490878c4fe323b868b61a9071 Mon Sep 17 00:00:00 2001 From: sn0w Date: Mon, 24 Dec 2018 23:21:16 +0100 Subject: [PATCH 02/21] Use compose 3.7 init instead of tini --- Dockerfile | 9 +-------- docker-compose.m4 | 3 ++- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5050680..c82d782 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,14 +10,7 @@ ENV LANG=C.UTF-8 # Register pseudo-entrypoint ADD ./entrypoint.sh / RUN chmod a+x /entrypoint.sh -CMD ["/entrypoint.sh"] - -# Set "real" entrypoint to an init system. -# TODO: Replace with --init when docker 18.06 is GA -ENV TINI_VERSION v0.18.0 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chmod +x /tini -ENTRYPOINT ["/tini", "--"] +ENTRYPOINT ["/entrypoint.sh"] # Get build dependencies RUN \ diff --git a/docker-compose.m4 b/docker-compose.m4 index d3c445d..22f286d 100644 --- a/docker-compose.m4 +++ b/docker-compose.m4 @@ -7,7 +7,7 @@ define(, <${upcase($1):?upcase($1)}>) define(, <${upcase($1):-$2}>) { - "version": "3", + "version": "3.7", ifdef(<__DOCKER_NETWORK>, < "networks": { @@ -91,6 +91,7 @@ define(, <${upcase($1):-$2}>) "env_fb(, )" ] }, + "init": true, "restart": "unless-stopped", "links": [ ifelse(__SCRIPT_DEPLOY_POSTGRES, true, <"db">) From 3adfa6642f86cf2f12b0efca8acc98cca2031a82 Mon Sep 17 00:00:00 2001 From: sn0w Date: Fri, 28 Dec 2018 01:44:33 +0100 Subject: [PATCH 03/21] Fix new config format --- .env.dist | 4 +++ Dockerfile | 57 ++++++++++++++++++++++++----------------- config.dist.yml | 17 ++++++------ config_parser/parser.rb | 18 +++++++++---- entrypoint.sh | 2 +- pleroma | 41 +++++++++++++++-------------- 6 files changed, 81 insertions(+), 58 deletions(-) diff --git a/.env.dist b/.env.dist index 113a0e8..ea38c3c 100644 --- a/.env.dist +++ b/.env.dist @@ -63,3 +63,7 @@ MIX_ENV=prod # The git tag, revision, or branch to check out on build PLEROMA_VERSION=develop + +# Domain to run at (only relevant for traefik mode) +PLEROMA_URL=coolsite.moe +PLEROMA_MEDIA_PROXY_URL=cdn.coolsite.moe diff --git a/Dockerfile b/Dockerfile index c82d782..300e083 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,61 +1,67 @@ FROM debian:9-slim -VOLUME /custom.d -EXPOSE 4000 - +# Set up environment ENV DEBIAN_FRONTEND=noninteractive ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -# Register pseudo-entrypoint -ADD ./entrypoint.sh / +# Prepare mounts and entrypoint +VOLUME /custom.d +VOLUME /conf + +COPY ./entrypoint.sh / RUN chmod a+x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] -# Get build dependencies +# Expose default pleroma port to host +EXPOSE 4000 + +# Get erlang, elixir, and dependencies RUN \ apt-get update \ && apt-get install -y --no-install-recommends apt-utils \ - && apt-get install -y --no-install-recommends git wget ca-certificates gnupg2 build-essential \ + && apt-get install -y --no-install-recommends git wget ca-certificates gnupg2 build-essential ruby \ \ && wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb \ && dpkg -i erlang-solutions_1.0_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends esl-erlang elixir \ - && apt-get install -y ruby \ \ && rm -rf /var/lib/apt/lists/* # Limit permissions -ARG DOCKER_UID -ARG DOCKER_GID -ARG PLEROMA_UPLOADS_PATH +ARG DOCKER_UID=1000 +ARG DOCKER_GID=1000 +ARG PLEROMA_UPLOADS_PATH=/uploads RUN \ groupadd --gid ${DOCKER_GID} pleroma \ && useradd -m -s /bin/bash --gid ${DOCKER_GID} --uid ${DOCKER_UID} pleroma \ && mkdir -p /custom.d $PLEROMA_UPLOADS_PATH \ - && chown -R pleroma:pleroma /custom.d $PLEROMA_UPLOADS_PATH + && chown -R pleroma:pleroma /custom.d /conf $PLEROMA_UPLOADS_PATH USER pleroma WORKDIR /home/pleroma -# Get pleroma +# Get pleroma sources RUN git clone --progress https://git.pleroma.social/pleroma/pleroma.git ./pleroma WORKDIR /home/pleroma/pleroma -# Get rebar/hex +# Bust the build cache (if needed) +# This works by setting an environment variable with the last +# used version/branch/tag/commitish/... which originates in the script. +# If the host doesn't have the required tool for "smart version detection" +# we'll just use the current timestamp here which forces a rebuild every time. +ARG __BUST_CACHE +ENV __BUST_CACHE $__BUST_CACHE + +# Get rebar and hex RUN \ mix local.hex --force \ && mix local.rebar --force -# Bust the build cache -ARG __BUST_CACHE -ENV __BUST_CACHE $__BUST_CACHE - # Fetch changes, checkout ARG PLEROMA_VERSION - RUN \ git fetch --all \ && git checkout $PLEROMA_VERSION \ @@ -66,9 +72,12 @@ RUN \ mix deps.get \ && mix compile -# Insert overrides and config helper -COPY --chown=pleroma:pleroma ./docker-config.exs /docker-config.exs -COPY --chown=pleroma:pleroma ./custom.d /home/pleroma/pleroma +# Prepare runtime config RUN \ - ln -s /docker-config.exs config/prod.secret.exs \ - && ln -s /docker-config.exs config/dev.secret.exs + ln -sf runtime-config.exs config/prod.secret.exs \ + && ln -sf runtime-config.exs config/dev.secret.exs + +# Insert overrides +COPY --chown=pleroma:pleroma ./custom.d /home/pleroma/pleroma + +# Recompiles at runtime if custom.d contained elixir code. diff --git a/config.dist.yml b/config.dist.yml index 14d94a9..5fabcb5 100644 --- a/config.dist.yml +++ b/config.dist.yml @@ -6,22 +6,20 @@ version: 1 # You can enter any config in here that you want. # Pleroma-Docker will try to translate it into elixir for you. # -# For example: +# is a special member for modifying the YAML->Elixir translation. +# When set to `Array` it causes the generation of a "keyed array" literal instead +# of multiple named parameters to `config`. # -# :foo: -# Bar.Baz: -# x: true +# is a special prefix that causes a string to be passed irectly without quoting. +# Useful for referencing modules like Ecto adapters or other symbols that are usually wrapped in yaml. # -# becomes `config :foo, Bar.Baz, x: true`. -# -# It is assumed that all config keys that have to be passed through to -# pleroma must start with an atom on the first layer (eg :pleroma). +# Remember to take a look at your config with `./pleroma config`. # app: # The loglevel to use in pleroma. :logger: - level: info + level: :info :pleroma: Pleroma.Repo: @@ -32,6 +30,7 @@ app: password: pleroma database: pleroma pool_size: 16 + adapter: Ecto.Adapters.Postgres Pleroma.Web.Endpoint: # Location where your instance will be reachable. diff --git a/config_parser/parser.rb b/config_parser/parser.rb index 96f1972..752f4d5 100755 --- a/config_parser/parser.rb +++ b/config_parser/parser.rb @@ -3,6 +3,14 @@ require 'yaml' require 'json' +def getval(val) + if val.is_a?(String) + val.start_with?('') ? val.delete('') : val.to_json() + else + val.to_json() + end +end + config = YAML.load_file(ARGV[0]) if config["version"] != 1 @@ -15,8 +23,8 @@ config["app"].each do |atom, content| content.each do |sub, settings| buf += "config :#{atom}, #{sub.is_a?(Symbol) ? ":#{sub}" : sub}" - if !settings.is_a? Hash - buf += ": #{settings.to_json}\n" + if !settings.is_a?(Hash) + buf += ": #{getval(settings)}\n" next end @@ -27,13 +35,13 @@ config["app"].each do |atom, content| buf += ", #{name}: [" value.each do |k, v| - buf += "#{k}: #{v.to_json}," + buf += "#{k}: #{getval(v)}," end - buf.chop! + buf.chop!() buf += "]" else - buf += ", #{name}: #{value.to_json}" + buf += ", #{name}: #{getval(value)}" end end diff --git a/entrypoint.sh b/entrypoint.sh index be5fc93..68c1d3b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,7 +4,7 @@ set -e set -x # Generate a config file -ruby /config/parser.rb /config/config.yml > runtime-config.exs +ruby /conf/parser.rb /conf/config.yml > config/runtime-config.exs # Recompile if needed if [[ ! -z "$RECOMPILE" ]]; then diff --git a/pleroma b/pleroma index cc00756..62e13c6 100755 --- a/pleroma +++ b/pleroma @@ -11,36 +11,38 @@ Usage: $0 [action] Actions: - build Rebuild the pleroma container. + build Rebuild the pleroma container. - dump Dump the generated docker-compose.yml to stdout. + config [file = config.yml] Print the generated pleroma config to stdout. - debug [bin] [args...] Launches a new pleroma container but uses \$bin instead of phx.server as entrypoint. - **Warning**: This is intended for debugging pleroma with tools like :debugger and :observer. - It thus forwards your X-Server into docker and temporarily fiddles with your xhost - access controls. If this is a security concern for you, please export NO_X_FORWARDING=1 - before launching a debugger session. + dump Dump the generated docker-compose.yml to stdout. - enter Spawn a shell inside the container for debugging/maintenance. - This command does not link to the postgres container. - If you need that use #debug instead. + debug [bin] [args...] Launches a new pleroma container but uses \$bin instead of phx.server as entrypoint. + **Warning**: This is intended for debugging pleroma with tools like :debugger and :observer. + It thus forwards your X-Server into docker and temporarily fiddles with your xhost + access controls. If this is a security concern for you, please export NO_X_FORWARDING=1 + before launching a debugger session. - logs Show the current container logs. + enter Spawn a shell inside the container for debugging/maintenance. + This command does not link to the postgres container. + If you need that use #debug instead. - mix [task] [args...] Run a mix task without entering the container. + logs Show the current container logs. - mod [file] Creates the file in custom.d and downloads the content from pleroma.social. - The download respects your \$PLEROMA_VERSION from .env. + mix [task] [args...] Run a mix task without entering the container. - passthrough / p [...] Pass any custom command to docker-compose. + mod [file] Creates the file in custom.d and downloads the content from pleroma.social. + The download respects your \$PLEROMA_VERSION from .env. - restart Executes #stop and #start respectively. + passthrough / p [...] Pass any custom command to docker-compose. - start / up Start pleroma and sibling services. + restart Executes #stop and #start respectively. - stop / down Stop pleroma and sibling services. + start / up Start pleroma and sibling services. - status / ps Show the current container status. + stop / down Stop pleroma and sibling services. + + status / ps Show the current container status. Environment: DEBUG can be used to modify the loglevel. @@ -91,6 +93,7 @@ load_env() { } action__build() { docker_compose build --build-arg __BUST_CACHE="$(date +%s)" server; } +action__config() { docker run --rm -t -i -v $(pwd):/mnt ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}"; } action__dump() { cat <(render_template); } action__enter() { docker_compose exec server ash -c 'cd /pleroma && ash'; } action__logs() { docker_compose logs -f; } From 37a999148168c73ebecd367f3feb634bc25e42d2 Mon Sep 17 00:00:00 2001 From: sn0w Date: Fri, 28 Dec 2018 01:51:44 +0100 Subject: [PATCH 04/21] Organize ./pleroma script --- pleroma | 233 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 146 insertions(+), 87 deletions(-) diff --git a/pleroma b/pleroma index 62e13c6..e5781a2 100755 --- a/pleroma +++ b/pleroma @@ -1,8 +1,150 @@ #!/bin/bash +######################################################### +# Options # +######################################################### + set -e set -o pipefail +######################################################### +# Globals # +######################################################### + +flags="" + +######################################################### +# Helpers # +######################################################### + +render_template() { + m4 $flags docker-compose.m4 | awk 'NF' +} + +docker_compose() { + docker-compose \ + -f <(render_template) \ + --project-directory . \ + "$@" +} + +load_env() { + if [[ ! -f .env ]]; then + echo "Please create a .env file first" + echo "(Copy .env.dist to .env for a template)" + exit 1 + fi + + while read -r line; do + if [[ "$line" == \#* ]] || [[ -z "$line" ]]; then + continue; + fi + + export "${line?}" + flags="-D__${line?} $flags" + done < .env +} + +######################################################### +# Subcommands # +######################################################### + +action__build() { + docker_compose build --build-arg __BUST_CACHE="$(date +%s)" server +} + +action__config() { + docker run --rm -t -i -v $(pwd):/mnt ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}" +} + +action__dump() { + cat <(render_template) +} + +action__enter() { + docker_compose exec server ash -c 'cd /pleroma && ash' +} + +action__logs() { + docker_compose logs -f $* +} + +action__mix() { + docker_compose exec server ash -c "cd /pleroma && mix $*" +} + +action__passthrough() { + docker_compose $* +} + +action__p() { + action__passthrough $* +} + +action__restart() { + action__stop + action__start +} + +action__start() { + docker_compose up --remove-orphans -d +} + +action__up() { + action__start +} + +action__stop() { + docker_compose down +} + +action__down() { + action__stop +} + +action__status() { + docker_compose ps +} + +action__ps() { + action__status +} + +action__debug() { + debug_mounts="-v $(pwd)/custom.d:/custom.d -v $(pwd)/debug.d/build:/home/pleroma/pleroma/_build -v $(pwd)/debug.d/deps:/home/pleroma/pleroma/deps" + + if [[ ! -d ./debug.d ]]; then + mkdir -p ./debug.d/{build,deps} + fi + + if [[ ! -d ./custom.d/lib ]]; then + mkdir -p ./custom.d/lib + fi + + action__stop + + docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts server bash -c 'cp -rvf /custom.d/* /home/pleroma/pleroma && mix deps.get' + + x_flags="" + if [[ $NO_X_FORWARDING != 1 ]]; then + x_flags="-e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix" + fi + + [[ $NO_X_FORWARDING == 1 ]] || xhost +local:root + docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts $x_flags server bash -c "cp -rvf /custom.d/* /home/pleroma/pleroma && $*" + [[ $NO_X_FORWARDING == 1 ]] || xhost -local:root +} + +action__mod() { + echo "Preparing 'custom.d/$1' for modding..." + install -D <(echo '') ./custom.d/$1 + wget -O ./custom.d/$1 https://git.pleroma.social/pleroma/pleroma/raw/$PLEROMA_VERSION/$1 +} + +######################################################### +# Help # +######################################################### + print_help() { echo " Pleroma Maintenance Script @@ -62,91 +204,9 @@ Contributing: " } -flags="" - -render_template() { - m4 $flags docker-compose.m4 | awk 'NF' -} - -docker_compose() { - docker-compose \ - -f <(render_template) \ - --project-directory . \ - "$@" -} - -load_env() { - if [[ ! -f .env ]]; then - echo "Please create a .env file first" - echo "(Copy .env.dist to .env for a template)" - exit 1 - fi - - while read -r line; do - if [[ "$line" == \#* ]] || [[ -z "$line" ]]; then - continue; - fi - - export "${line?}" - flags="-D__${line?} $flags" - done < .env -} - -action__build() { docker_compose build --build-arg __BUST_CACHE="$(date +%s)" server; } -action__config() { docker run --rm -t -i -v $(pwd):/mnt ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}"; } -action__dump() { cat <(render_template); } -action__enter() { docker_compose exec server ash -c 'cd /pleroma && ash'; } -action__logs() { docker_compose logs -f; } -action__mix() { docker_compose exec server ash -c "cd /pleroma && mix $*"; } -action__passthrough() { docker_compose $*; } -action__p() { action__passthrough $*; } - -action__restart() { action__stop; action__start; } - -action__start() { docker_compose up --remove-orphans -d; } -action__up() { action__start; } - -action__stop() { docker_compose down; } -action__down() { action__stop; } - -action__status() { docker_compose ps; } -action__ps() { action__status; } - -### -# This function rips out the mix caches from the container -# in order to speed up rebuilds during debugging/modding sessions. -# To persist the changes, the user still needs to rebuild the container. -### -action__debug() { - debug_mounts="-v $(pwd)/custom.d:/custom.d -v $(pwd)/debug.d/build:/home/pleroma/pleroma/_build -v $(pwd)/debug.d/deps:/home/pleroma/pleroma/deps" - - if [[ ! -d ./debug.d ]]; then - mkdir -p ./debug.d/{build,deps} - fi - - if [[ ! -d ./custom.d/lib ]]; then - mkdir -p ./custom.d/lib - fi - - action__stop - - docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts server bash -c 'cp -rvf /custom.d/* /home/pleroma/pleroma && mix deps.get' - - x_flags="" - if [[ $NO_X_FORWARDING != 1 ]]; then - x_flags="-e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix" - fi - - [[ $NO_X_FORWARDING == 1 ]] || xhost +local:root - docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts $x_flags server bash -c "cp -rvf /custom.d/* /home/pleroma/pleroma && $*" - [[ $NO_X_FORWARDING == 1 ]] || xhost -local:root -} - -action__mod() { - echo "Preparing 'custom.d/$1' for modding..." - install -D <(echo '') ./custom.d/$1 - wget -O ./custom.d/$1 https://git.pleroma.social/pleroma/pleroma/raw/$PLEROMA_VERSION/$1 -} +######################################################### +# Main # +######################################################### # Check if there is any command at all if [[ -z "$1" ]]; then @@ -177,8 +237,7 @@ fi # Parse .env load_env -# Guess function name of current command -# and then check for it's existance. +# Guess function name of current command and call it if present func="action__${1}" if type -t $func 2>&1 1>/dev/null; then From 94de1ce2b4c5de164838af801936e1137a8a05ca Mon Sep 17 00:00:00 2001 From: sn0w Date: Fri, 28 Dec 2018 01:52:39 +0100 Subject: [PATCH 05/21] Add config.yml to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aa4b1f7..6bbc7ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ data .env +config.yml custom.d/ !custom.d/.gitkeep docker-compose.yml From f92c63d76d564355fa3e28b92ec37433f42e91b1 Mon Sep 17 00:00:00 2001 From: sn0w Date: Fri, 28 Dec 2018 01:54:12 +0100 Subject: [PATCH 06/21] Add todo in parser --- config_parser/parser.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config_parser/parser.rb b/config_parser/parser.rb index 752f4d5..e5d718a 100755 --- a/config_parser/parser.rb +++ b/config_parser/parser.rb @@ -1,5 +1,12 @@ #!/usr/bin/env ruby +# +# TODO: Write a better translator. +# This is just rough text replacement right now +# but it's already way better than the exs stuff +# we had durin the .env files. +# + require 'yaml' require 'json' From 6c608941996d2f305778e708b6b51fe2cbe838c0 Mon Sep 17 00:00:00 2001 From: sn0w Date: Fri, 28 Dec 2018 02:05:31 +0100 Subject: [PATCH 07/21] Add license and reset README for rewrite --- LICENSE | 22 ++++++++ README.md | 162 +----------------------------------------------------- 2 files changed, 25 insertions(+), 159 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bd3a56b --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2017-2018, sn0w +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 9d03d26..1c062cd 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ This repository dockerizes it for easier deployment.
```cpp -#include -#include +#include /* * This repository comes with ABSOLUTELY NO WARRANTY @@ -24,14 +23,6 @@ This repository dockerizes it for easier deployment.
-## Features - -- 100% generic -- Everything is customizable -- Zero special host dependencies -- Configuration is not compile-time -- "It just works" - ## Alternatives If this setup is a bit overwhelming there are a lot of other great dockerfiles @@ -41,153 +32,6 @@ or guides from the community. A few are linked below. This list is not exhaustiv - [RX14/iscute.moe](https://github.com/RX14/kurisu.rx14.co.uk/blob/master/services/iscute.moe/pleroma/Dockerfile) - [rysiek/docker-pleroma](https://git.pleroma.social/rysiek/docker-pleroma) -## Installing Pleroma +## Docs -- Clone this repository -- Copy `.env.dist` to `.env` -- Edit `.env` (see "Configuring Pleroma" section below) -- Run `./pleroma build` and `./pleroma start` -- Profit! - -## Updating Pleroma - -Just run `./pleroma build` again and `./pleroma start` afterwards. - -You don't need to shutdown pleroma while compiling the new release. - -Every time you run `./pleroma build` the script will fetch all upstream changes and checkout `PLEROMA_VERSION`. -This means that setting `PLEROMA_VERSION` to a branch enables rolling-release updates while setting -it to a tag or commit-hash pins the version. - -## Maintaining Pleroma - -Pleroma maintenance is usually done with premade mix tasks.
-You can run these tasks using `./pleroma mix [task] [arguments...]`.
-If you need to fix some bigger issues you can also spawn a shell using `./pleroma enter`. - -## Customizing Pleroma - -Just add your customizations (and their folder structure) to `custom.d`.
-They will be mounted and symlinked into the right place when the container starts.
-You can even replace/patch pleroma's code with this, because the project is recompiled at startup.
- -In general: Prepending `custom.d/` to pleroma's customization guides should work all the time.
-Check them out in the [official pleroma wiki](https://git.pleroma.social/pleroma/pleroma/wikis/home). - -For example: A custom thumbnail now goes into `custom.d/priv/static/instance/thumbnail.jpeg` instead of `priv/static/instance/thumbnail.jpeg`. - -Note: Since `custom.d` needs to be accessible at runtime by the pleroma process, the container will automatically chown these files to `$UID:$GID` from your `.env` file. - -## Configuring Pleroma - -pleroma-docker tries to stay out of your way as much as possible while providing -a good experience for both you and your users. It thus supports multiple -"operation modes" and quite some config variables which you can mix and match. - -This guide will explain some of the tricky `.env` file parts as detailed as possible (but you should still read the comments in there). - -Since this setup [injects code](https://glitch.sh/sn0w/pleroma-docker/blob/master/docker-config.exs) into pleroma that moves it's configuration into the environment (ref ["The Twelve-Factor App"](https://12factor.net/)), -the built image is 100% reusable and can be shared/replicated across multiple hosts. -To do that just run `./pleroma build` as usual and then tag your image to whatever you want. -Just make sure to start the replicated container with `env_file:` or all required `-e` pairs. - -#### Storing Data - -Currently all data is stored in subfolders of `DOCKER_DATADIR` which will be bind-mounted into the container by docker. - -We'll evaluate named volumes as an option in the future but they're currently not supported. - -#### Database (`SCRIPT_DEPLOY_POSTGRES`) - -Values: `true` / `false` - -By default pleroma-docker deploys a postgresql container and links it to pleroma's container as a zero-config data store. If you already have a postgres database or want to host postgres on a physically different machine set this value to `false`. Make sure to set the `POSTGRES_*` variables when doing that. - -#### Reverse Proxy (`SCRIPT_USE_PROXY`) - -Values: `traefik` / `nginx` / `manual` - -Pleroma is usually run behind a reverse-proxy. -Pleroma-docker gives you multiple options here. - -##### Traefik - -In traefik-mode we will generate a pleroma container with traefik labels. -These will be picked up at runtime to dynamically create a reverse-proxy -configuration. This should 'just work' if `watch=true` and `exposedByDefault=false` are set in the `[docker]` section of your `traefik.conf`. SSL will also 'just work' once you add a matching `[[acme.domains]]` entry. - -##### NGINX - -In nginx-mode we will generate a bare nginx container that is linked to the -pleroma container. The nginx container is absolutely unmodified and expects to -be configured by you. The nginx file in [Pleroma's Repository](https://git.pleroma.social/pleroma/pleroma/blob/develop/installation/pleroma.nginx) is a good starting point. - -We will mount your configs like this: -``` -custom.d/server.nginx -> /etc/nginx/nginx.conf -custom.d/vhost.nginx -> /etc/nginx/conf.d/pleroma.conf -``` - -To reach your pleroma container from inside nginx use `proxy_pass http://pleroma:4000;`. - -Set `SCRIPT_PORT_HTTP` and `SCRIPT_PORT_HTTPS` to the ports you want to listen on. -Specify the ip to bind to in `SCRIPT_BIND_IP`. These values are required. - -The container only listens on `SCRIPT_PORT_HTTPS` if `SCRIPT_ENABLE_SSL` is `true`. - -##### Apache / httpd - -Just like nginx-mode this starts an unmodified apache server that expects to be -configured by you. Again [Pleroma's Config](https://git.pleroma.social/pleroma/pleroma/blob/develop/installation/pleroma-apache.conf) is a good starting point. - -We will mount your configs like this: -``` -custom.d/server.httpd -> /usr/local/apache2/conf/httpd.conf -custom.d/vhost.httpd -> /usr/local/apache2/conf/extra/httpd-vhosts.conf -``` - -To reach your pleroma container from inside apache use `ProxyPass [loc] http://pleroma:4000/`. - -Again setting `SCRIPT_PORT_HTTP`, `SCRIPT_PORT_HTTPS` and `SCRIPT_BIND_IP` is required. - -The container only listens on `SCRIPT_PORT_HTTPS` if `SCRIPT_ENABLE_SSL` is `true`. - -##### Manual - -In manual mode we do not create any reverse proxy for you. -You'll have to figure something out on your own. - -This mode also doesn't bind to any IP or port. -You'll have to forward something to the container's IP. - -#### SSL (`SCRIPT_ENABLE_SSL`) - -Values: `true` / `false` - -If you want to use SSL with your Apache or NGINX containers you'll need a -certificate. Certificates need to be placed into `custom.d` and will be -bind-mounted into the server's container at runtime. - -We will mount your certs like this: -``` -custom.d/ssl.crt -> /ssl/ssl.crt -custom.d/ssl.key -> /ssl/ssl.key -``` - -You can reference them in Apache like this: -```apache - - SSLEngine on - SSLCertificateFile "/ssl/ssl.crt" - SSLCertificateKeyFile "/ssl/ssl.key" - -``` - -And in NGINX like this: -```nginx -listen 443 ssl; -ssl_certificate /ssl/ssl.crt; -ssl_certificate_key /ssl/ssl.key; -``` - -In traefik-mode and manual-mode these files and the `SCRIPT_ENABLE_SSL` value are ignored. +coming soon tm From e8855ead76efbe7ba3c66efb2cc46b34f0d3b239 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 30 Dec 2018 14:29:51 +0100 Subject: [PATCH 08/21] Move entrypoint addition I have slow internet ok --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 300e083..37c78a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,14 +5,10 @@ ENV DEBIAN_FRONTEND=noninteractive ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -# Prepare mounts and entrypoint +# Prepare mounts VOLUME /custom.d VOLUME /conf -COPY ./entrypoint.sh / -RUN chmod a+x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] - # Expose default pleroma port to host EXPOSE 4000 @@ -29,6 +25,11 @@ RUN \ \ && rm -rf /var/lib/apt/lists/* +# Add entrypoint +COPY ./entrypoint.sh / +RUN chmod a+x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + # Limit permissions ARG DOCKER_UID=1000 ARG DOCKER_GID=1000 From 3c512c77880418cfad769bb9284dd8eb8d3326d0 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 30 Dec 2018 14:30:36 +0100 Subject: [PATCH 09/21] Move compilation into entrypoint; Call entrypoint on build --- Dockerfile | 5 +---- entrypoint.sh | 13 +++++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 37c78a7..f2f9005 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,8 +70,7 @@ RUN \ # Precompile RUN \ - mix deps.get \ - && mix compile + NO_CONFIG=1 COMPILE_ONLY=1 /entrypoint.sh # Prepare runtime config RUN \ @@ -80,5 +79,3 @@ RUN \ # Insert overrides COPY --chown=pleroma:pleroma ./custom.d /home/pleroma/pleroma - -# Recompiles at runtime if custom.d contained elixir code. diff --git a/entrypoint.sh b/entrypoint.sh index 68c1d3b..a2c2a8e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,15 +3,20 @@ set -e set -x -# Generate a config file -ruby /conf/parser.rb /conf/config.yml > config/runtime-config.exs +if [[ -z "$NO_CONFIG" ]]; then + ruby /conf/parser.rb /conf/config.yml > config/runtime-config.exs +fi -# Recompile if needed -if [[ ! -z "$RECOMPILE" ]]; then +if [[ -n "$COMPILE_ONLY" ]]; then mix deps.get mix compile + exit 0 fi +# Assume that dependencies are compiled and ready to go. +# Remove this assumption when https://github.com/erlang/rebar3/issues/1627 is fixed. +mix compile + # Migrate db mix ecto.create mix ecto.migrate From 074ea0ef8f044ee17957e520ea160d92b2158081 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 30 Dec 2018 14:31:32 +0100 Subject: [PATCH 10/21] Add new helper methods --- pleroma | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pleroma b/pleroma index e5781a2..31e478b 100755 --- a/pleroma +++ b/pleroma @@ -17,6 +17,20 @@ flags="" # Helpers # ######################################################### +has_command() { + if command -v 1>/dev/null 2>&1 "$1"; then + return 0 + else + return 1 + fi +} + +require_command() { + if ! has_command "$1"; then + printf "\nError: This action requires the command '%s' in your PATH.\n" "$1" + exit 1 + fi +} render_template() { m4 $flags docker-compose.m4 | awk 'NF' } @@ -45,6 +59,27 @@ load_env() { done < .env } +download_file() { # $1: source, $2: target + if has_command curl; then + curl -sSL "$1" -o "$2" + elif has_command wget; then + wget "$1" -O "$2" + else + printf "\nError: This action requires either curl or wget in your PATH.\n" + exit 1 + fi +} + +request_file_content() { # $1: source + if has_command curl; then + curl -sSL "$1" + elif has_command wget; then + wget "$1" -O- 2>/dev/null + else + printf "\nError: This action requires either curl or wget in your PATH.\n" + exit 1 + fi +} ######################################################### # Subcommands # ######################################################### From b96154acc8e2bea4b41b0264c9de6a4f236aeb16 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 30 Dec 2018 14:33:42 +0100 Subject: [PATCH 11/21] Modernize pleroma script - Require commands before using them - Use constants for git url - Remove old leftovers - Re-add dialog-powered #mod --- pleroma | 94 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/pleroma b/pleroma index 31e478b..3758331 100755 --- a/pleroma +++ b/pleroma @@ -11,6 +11,14 @@ set -o pipefail # Globals # ######################################################### +readonly GITLAB_URI="https://git.pleroma.social" +readonly PREFIX_API="api/v4/projects/pleroma%2Fpleroma/repository" +readonly ENDPOINT_FILE="pleroma/pleroma/raw" +readonly ENDPOINT_LIST="pleroma/pleroma/files" +readonly ENDPOINT_TAG="$PREFIX_API/tags" +readonly ENDPOINT_BLOB="$PREFIX_API/blobs" +readonly ENDPOINT_BRANCH="$PREFIX_API/branches" + flags="" ######################################################### @@ -31,11 +39,17 @@ require_command() { exit 1 fi } + render_template() { + require_command m4 + require_command awk + m4 $flags docker-compose.m4 | awk 'NF' } docker_compose() { + require_command docker-compose + docker-compose \ -f <(render_template) \ --project-directory . \ @@ -80,6 +94,7 @@ request_file_content() { # $1: source exit 1 fi } + ######################################################### # Subcommands # ######################################################### @@ -89,7 +104,9 @@ action__build() { } action__config() { - docker run --rm -t -i -v $(pwd):/mnt ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}" + require_command docker + + docker run --rm -t -i -v "$(pwd):/mnt" ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}" } action__dump() { @@ -97,23 +114,23 @@ action__dump() { } action__enter() { - docker_compose exec server ash -c 'cd /pleroma && ash' + docker_compose exec server sh -c 'cd ~/pleroma && bash' } action__logs() { - docker_compose logs -f $* + docker_compose logs "$@" } action__mix() { - docker_compose exec server ash -c "cd /pleroma && mix $*" + docker_compose exec server sh -c "cd ~/pleroma && mix $*" } action__passthrough() { - docker_compose $* + docker_compose "$@" } action__p() { - action__passthrough $* + action__passthrough "$@" } action__restart() { @@ -146,7 +163,14 @@ action__ps() { } action__debug() { - debug_mounts="-v $(pwd)/custom.d:/custom.d -v $(pwd)/debug.d/build:/home/pleroma/pleroma/_build -v $(pwd)/debug.d/deps:/home/pleroma/pleroma/deps" + require_command xhost + + local debug_mounts + debug_mounts=" + -v $(pwd)/custom.d:/custom.d \ + -v $(pwd)/debug.d/build:/home/pleroma/pleroma/_build \ + -v $(pwd)/debug.d/deps:/home/pleroma/pleroma/deps \ + " if [[ ! -d ./debug.d ]]; then mkdir -p ./debug.d/{build,deps} @@ -158,22 +182,54 @@ action__debug() { action__stop - docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts server bash -c 'cp -rvf /custom.d/* /home/pleroma/pleroma && mix deps.get' + docker_compose run --rm -u pleroma -w /home/pleroma/pleroma "$debug_mounts" server bash -c 'cp -rvf /custom.d/* /home/pleroma/pleroma && mix deps.get' - x_flags="" + local x_flags="" if [[ $NO_X_FORWARDING != 1 ]]; then x_flags="-e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix" fi [[ $NO_X_FORWARDING == 1 ]] || xhost +local:root - docker_compose run --rm -u pleroma -w /home/pleroma/pleroma $debug_mounts $x_flags server bash -c "cp -rvf /custom.d/* /home/pleroma/pleroma && $*" + docker_compose run --rm -u pleroma -w /home/pleroma/pleroma "$debug_mounts" "$x_flags" server bash -c "cp -rvf /custom.d/* /home/pleroma/pleroma && $*" [[ $NO_X_FORWARDING == 1 ]] || xhost -local:root } action__mod() { - echo "Preparing 'custom.d/$1' for modding..." - install -D <(echo '') ./custom.d/$1 - wget -O ./custom.d/$1 https://git.pleroma.social/pleroma/pleroma/raw/$PLEROMA_VERSION/$1 + require_command dialog + require_command jq + require_command curl + + if [[ ! -d ./debug.d ]]; then + mkdir ./debug.d + fi + + if [[ ! -f ./debug.d/mod_files.json ]] || [[ -n "$(find ./debug.d/mod_files.json -mmin +5)" ]]; then + curl -sSL -# "$GITLAB_URI/$ENDPOINT_LIST/$PLEROMA_VERSION?format=json" > ./debug.d/mod_files.json + + if [[ -f ./debug.d/mod_files.lst ]]; then + rm ./debug.d/mod_files.lst + fi + + jq -r 'map("\(.)\n") | add' <./debug.d/mod_files.json >./debug.d/mod_files.lst + fi + + if [[ -f ./debug.d/mod_files.lst ]] && [[ -r ./debug.d/mod_files.lst ]]; then + choices="" + + while read -r candidate; do + choices="$choices $candidate $(echo "$candidate" | rev | cut -d/ -f1 | rev)" + done <<< "$(grep -E ".*$1.*" <./debug.d/mod_files.lst)" + + res=$(mktemp) + dialog --menu "Select the file you want to modify:" 35 80 30 $choices 2>"$res" + choice=$(cat "$res") + + install -D <(echo '') "./custom.d/$choice" + curl -sSL -# "$GITLAB_URI/$ENDPOINT_FILE/$PLEROMA_VERSION/$choice" > "./custom.d/$choice" + else + install -D <(echo '') "./custom.d/$1" + curl -sSL -# "$GITLAB_URI/$ENDPOINT_FILE/$PLEROMA_VERSION/$1" > "./custom.d/$1" + fi } ######################################################### @@ -250,18 +306,18 @@ if [[ -z "$1" ]]; then fi # Check for SHOPTs -if [[ ! -z "$SHOPT" ]]; then +if [[ -n "$SHOPT" ]]; then for opt in $SHOPT; do if [[ $opt =~ ":" ]]; then - set -o ${opt//-o:/} + set -o "${opt//-o:/}" else - set $opt + set "$opt" fi done fi # Check for DEBUG -if [[ ! -z "$DEBUG" ]]; then +if [[ -n "$DEBUG" ]]; then if [[ $DEBUG == 1 ]]; then export DEBUG_COMMANDS=1 elif [[ $DEBUG == 2 ]]; then @@ -275,10 +331,10 @@ load_env # Guess function name of current command and call it if present func="action__${1}" -if type -t $func 2>&1 1>/dev/null; then +if type -t "$func" 1>/dev/null 2>&1; then shift [[ $DEBUG != 1 ]] || set -x - $func $@ + $func "$@" { [[ $DEBUG != 1 ]] || set +x; } 2>/dev/null else print_help From 04432bfc8a553a355614e0bd584a8d20c6358a90 Mon Sep 17 00:00:00 2001 From: sn0w Date: Wed, 9 Jan 2019 21:23:53 +0100 Subject: [PATCH 12/21] ok this is emberassing Fixes the caching problem and configuration madness. Startup on a migrated database takes about 1 to 3 seconds instead of minutes now. I don't know what went through my head when building the old stuff. Let's just pretend all commits up until here never happended. We good? nice~ --- Dockerfile | 18 ++----- config.dist.exs | 14 +++++ config.dist.yml | 110 ---------------------------------------- config.exs | 14 +++++ config_parser/parser.rb | 59 --------------------- docker-compose.m4 | 5 +- entrypoint.sh | 9 ++-- pleroma | 76 ++++++++++++++++----------- 8 files changed, 85 insertions(+), 220 deletions(-) create mode 100644 config.dist.exs delete mode 100644 config.dist.yml create mode 100644 config.exs delete mode 100755 config_parser/parser.rb diff --git a/Dockerfile b/Dockerfile index f2f9005..ea9f50d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ FROM debian:9-slim # Set up environment +ENV MIX_ENV=prod ENV DEBIAN_FRONTEND=noninteractive ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 # Prepare mounts VOLUME /custom.d -VOLUME /conf # Expose default pleroma port to host EXPOSE 4000 @@ -39,7 +39,7 @@ RUN \ groupadd --gid ${DOCKER_GID} pleroma \ && useradd -m -s /bin/bash --gid ${DOCKER_GID} --uid ${DOCKER_UID} pleroma \ && mkdir -p /custom.d $PLEROMA_UPLOADS_PATH \ - && chown -R pleroma:pleroma /custom.d /conf $PLEROMA_UPLOADS_PATH + && chown -R pleroma:pleroma /custom.d $PLEROMA_UPLOADS_PATH USER pleroma WORKDIR /home/pleroma @@ -56,11 +56,6 @@ WORKDIR /home/pleroma/pleroma ARG __BUST_CACHE ENV __BUST_CACHE $__BUST_CACHE -# Get rebar and hex -RUN \ - mix local.hex --force \ - && mix local.rebar --force - # Fetch changes, checkout ARG PLEROMA_VERSION RUN \ @@ -70,12 +65,9 @@ RUN \ # Precompile RUN \ - NO_CONFIG=1 COMPILE_ONLY=1 /entrypoint.sh - -# Prepare runtime config -RUN \ - ln -sf runtime-config.exs config/prod.secret.exs \ - && ln -sf runtime-config.exs config/dev.secret.exs + cp ./config/dev.exs ./config/prod.secret.exs \ + && BUILDTIME=1 /entrypoint.sh \ + && rm ./config/prod.secret.exs # Insert overrides COPY --chown=pleroma:pleroma ./custom.d /home/pleroma/pleroma diff --git a/config.dist.exs b/config.dist.exs new file mode 100644 index 0000000..046b38e --- /dev/null +++ b/config.dist.exs @@ -0,0 +1,14 @@ +use Mix.Config + +# pleroma/pleroma/pleroma are the default credentials for the +# managed database container. "db" is the default interlinked hostname. +# You shouldn't need to change this unless you modifed .env +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: "pleroma", + password: "pleroma", + database: "pleroma", + hostname: "db", + pool_size: 10 + +# vvv Your awesome config options go here vvv diff --git a/config.dist.yml b/config.dist.yml deleted file mode 100644 index 5fabcb5..0000000 --- a/config.dist.yml +++ /dev/null @@ -1,110 +0,0 @@ -version: 1 - -# -# Pleroma settings -# -# You can enter any config in here that you want. -# Pleroma-Docker will try to translate it into elixir for you. -# -# is a special member for modifying the YAML->Elixir translation. -# When set to `Array` it causes the generation of a "keyed array" literal instead -# of multiple named parameters to `config`. -# -# is a special prefix that causes a string to be passed irectly without quoting. -# Useful for referencing modules like Ecto adapters or other symbols that are usually wrapped in yaml. -# -# Remember to take a look at your config with `./pleroma config`. -# - -app: - # The loglevel to use in pleroma. - :logger: - level: :info - - :pleroma: - Pleroma.Repo: - # Credentials for your database. - # You should leave this as-is if you want to use the managed db container. - hostname: db - username: pleroma - password: pleroma - database: pleroma - pool_size: 16 - adapter: Ecto.Adapters.Postgres - - Pleroma.Web.Endpoint: - # Location where your instance will be reachable. - url: - : Array - scheme: https - host: coolsite.moe - port: 443 - - # Base for your secret keys. - # Better make this random. - secret_key_base: asdf0815 - - Pleroma.Upload: - # Where to store your uploads. - # You should probably leave this as-is. - # /uploads will be mounted into ::docker::datadir. - uploads: /uploads - - # Remove metadata from uploads? - strip_exif: true - - :chat: - # Enable chat functionality? - enabled: true - - :media_proxy: - # Enable the proxy? - enabled: true - - # Emit a 302 to the original resource when uncached? - redirect_on_failure: true - - # Where your proxy is reachable - base_url: https://media.coolsite.moe - - :instance: - # The name of your instance. - name: super cool stuff club - - # Short description of your instance - description: we do super cool stuff in super cool stuff club - - # The admin's email address. - email: admin@coolsite.moe - - # How many chars a notice may have at max. - limit: 4096 - - # May new members sign up? - registrations_open: true - - # Allow connections to other instances? - # (Turn this off for testing) - federating: true - - # The rewrite policies / quarantines to enable. - # This is a powerful feature which should be used with care. - # Take a look at https://git.pleroma.social/pleroma/pleroma/wikis/Message%20rewrite%20facility%20configuration%20(how%20to%20block%20instances). - # Then uncomment only the things you really need - - # quarantined_instances: - # - badguys.moe - - # rewrite_policy: - # - Pleroma.Web.ActivityPub.MRF.SimplePolicy - - # :mrf_simple: - # media_removal: - # - illegalporn.biz - # media_nsfw: - # - porn.biz - # - porn.business - # reject: - # - spam.com - # federated_timeline_removal: - # - spam.university diff --git a/config.exs b/config.exs new file mode 100644 index 0000000..046b38e --- /dev/null +++ b/config.exs @@ -0,0 +1,14 @@ +use Mix.Config + +# pleroma/pleroma/pleroma are the default credentials for the +# managed database container. "db" is the default interlinked hostname. +# You shouldn't need to change this unless you modifed .env +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: "pleroma", + password: "pleroma", + database: "pleroma", + hostname: "db", + pool_size: 10 + +# vvv Your awesome config options go here vvv diff --git a/config_parser/parser.rb b/config_parser/parser.rb deleted file mode 100755 index e5d718a..0000000 --- a/config_parser/parser.rb +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env ruby - -# -# TODO: Write a better translator. -# This is just rough text replacement right now -# but it's already way better than the exs stuff -# we had durin the .env files. -# - -require 'yaml' -require 'json' - -def getval(val) - if val.is_a?(String) - val.start_with?('') ? val.delete('') : val.to_json() - else - val.to_json() - end -end - -config = YAML.load_file(ARGV[0]) - -if config["version"] != 1 - raise "Incompatible config version (#{config["version"]} != 1)" -end - -buf = "use Mix.Config\n\n" - -config["app"].each do |atom, content| - content.each do |sub, settings| - buf += "config :#{atom}, #{sub.is_a?(Symbol) ? ":#{sub}" : sub}" - - if !settings.is_a?(Hash) - buf += ": #{getval(settings)}\n" - next - end - - settings.each do |name, value| - if value.is_a?(Hash) && value[""] == "Array" - value.delete("") - - buf += ", #{name}: [" - - value.each do |k, v| - buf += "#{k}: #{getval(v)}," - end - buf.chop!() - - buf += "]" - else - buf += ", #{name}: #{getval(value)}" - end - end - - buf += "\n" - end -end - -puts buf diff --git a/docker-compose.m4 b/docker-compose.m4 index 22f286d..aacbc0f 100644 --- a/docker-compose.m4 +++ b/docker-compose.m4 @@ -100,9 +100,8 @@ define(, <${upcase($1):-$2}>) "env_fb(, )" ], "volumes": [ - "./custom.d:/custom.d", - "./config.yml:/conf/config.yml:ro", - "./config_parser/parser.rb:/conf/parser.rb:ro", + "./custom.d:/custom.d:ro", + "./config.exs:/home/pleroma/pleroma/config/prod.secret.exs:ro", "env_inline()/uploads:env_inline_fb(, )" ], "labels": [ diff --git a/entrypoint.sh b/entrypoint.sh index a2c2a8e..422e7fb 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,18 +3,15 @@ set -e set -x -if [[ -z "$NO_CONFIG" ]]; then - ruby /conf/parser.rb /conf/config.yml > config/runtime-config.exs -fi +if [[ -n "$BUILDTIME" ]]; then + mix local.rebar --force + mix local.hex --force -if [[ -n "$COMPILE_ONLY" ]]; then mix deps.get mix compile exit 0 fi -# Assume that dependencies are compiled and ready to go. -# Remove this assumption when https://github.com/erlang/rebar3/issues/1627 is fixed. mix compile # Migrate db diff --git a/pleroma b/pleroma index 3758331..fa4f39d 100755 --- a/pleroma +++ b/pleroma @@ -40,6 +40,22 @@ require_command() { fi } +require_file() { + if [[ ! -f $1 ]]; then + echo "File missing: '$1' (Example at: '$2')" + FILE_FAILED=1 + fi +} + +throw_file_errors() { + if [[ -n "$FILE_FAILED" ]]; then + echo "" + echo "Please create the missing files first." + echo "The script will now exit." + exit 1 + fi +} + render_template() { require_command m4 require_command awk @@ -57,12 +73,6 @@ docker_compose() { } load_env() { - if [[ ! -f .env ]]; then - echo "Please create a .env file first" - echo "(Copy .env.dist to .env for a template)" - exit 1 - fi - while read -r line; do if [[ "$line" == \#* ]] || [[ -z "$line" ]]; then continue; @@ -103,12 +113,6 @@ action__build() { docker_compose build --build-arg __BUST_CACHE="$(date +%s)" server } -action__config() { - require_command docker - - docker run --rm -t -i -v "$(pwd):/mnt" ruby:alpine sh -c "cd /mnt && ruby config_parser/parser.rb ${1:-config.yml}" -} - action__dump() { cat <(render_template) } @@ -241,12 +245,10 @@ print_help() { Pleroma Maintenance Script Usage: - $0 [action] + $0 [action] [action-args...] Actions: - build Rebuild the pleroma container. - - config [file = config.yml] Print the generated pleroma config to stdout. + build (Re)build the pleroma container. dump Dump the generated docker-compose.yml to stdout. @@ -284,8 +286,6 @@ Environment: SHOPT can be used to modify shell options. Pass a list of options to this variable like SHOPT='-x -e'. - -e is always on unless you set it to +e. - For setting long options with -o use a colon (:) instead of a space to seperate the option from -o. For example: SHOPT='-x -e -o:pipefail'. @@ -325,18 +325,36 @@ if [[ -n "$DEBUG" ]]; then fi fi +# Check if the option is "help" +case "$1" in + "help"|"h"|"--help"|"-h"|"-?") + print_help + exit 0 + ;; +esac + +# Check if the called command exists +func="action__${1}" +if ! type -t "$func" 1>/dev/null 2>&1; then + echo "Unknown flag or subcommand." + echo "Try '$0 help'" + exit 1 +fi + +# Fail if mandatory files are missing +require_file ".env" ".env.dist" +require_file "config.exs" "config.dist.exs" +throw_file_errors + # Parse .env load_env -# Guess function name of current command and call it if present -func="action__${1}" +# Handle DEBUG=2 +[[ $DEBUG != 1 ]] || set -x -if type -t "$func" 1>/dev/null 2>&1; then - shift - [[ $DEBUG != 1 ]] || set -x - $func "$@" - { [[ $DEBUG != 1 ]] || set +x; } 2>/dev/null -else - print_help - exit 1 -fi +# Jump to called function +shift +$func "$@" + +# Disable debug mode +{ [[ $DEBUG != 1 ]] || set +x; } 2>/dev/null From b65e565b4ba8d0946321b0ec131838c4e76fe190 Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 01:06:29 +0100 Subject: [PATCH 13/21] Rewrite docs --- README.md | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c062cd..62d04fe 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,10 @@ This repository dockerizes it for easier deployment. * This repository comes with ABSOLUTELY NO WARRANTY * * I am not responsible for burning servers, angry users, fedi drama, - * thermonuclear war, or you getting fired because your boss saw your - * NSFW posts. Please do some research if you have any concerns about included - * features or the software used by this script before using it. + * thermonuclear war, or you getting fired because your boss saw your NSFW posts. + * Please do some research if you have any concerns about included + * features or the software used by this script ***before*** using it. + * * You are choosing to use this setup, and if you point the finger at me for * messing up your instance, I will laugh at you. */ @@ -34,4 +35,149 @@ or guides from the community. A few are linked below. This list is not exhaustiv ## Docs -coming soon tm +### Installation + +- Clone this repository +- Create a `config.exs` and `.env` +- Run `./pleroma build` and `./pleroma up` +- Profit! + +### Updates + +Run `./pleroma build` again and start the updated image with `./pleroma up`. + +You don't need to stop your pleroma server for either of those commands. + +### Maintenance + +Pleroma maintenance is usually done with mix tasks. +You can run these tasks in your running pleroma server using `./pleroma mix [task] [arguments...]`. +If you need to fix some bigger issues you can also spawn a shell with `./pleroma enter`. + +### Customization + +Add your customizations (and their folder structure) to `custom.d/`. +They will be mounted and symlinked into the right place when the container starts. +You can even replace/patch pleroma’s code with this, because the project is recompiled at startup if needed. + +In general: Prepending `custom.d/` to pleroma’s customization guides should work all the time.
+Check them out in the official pleroma wiki. + +For example: A custom thumbnail now goes into `custom.d/priv/static/instance/thumbnail.jpeg` instead of `priv/static/instance/thumbnail.jpeg`. + +### Patches + +Works exactly like customization, but we have a neat little helper here. + +Use `./pleroma mod [regex]` to mod any file that ships with pleroma, without having to type the complete path.
+ +### Configuration + +All the pleroma options that you put into your `*.secret.exs` now go into `config.exs`. + +`.env` stores config values that need to be known at orchestration time.
+They should be self-explaining but here's some bonus info on important ones: + +#### Data Storage (`DOCKER_DATADIR`) + +A folder that will be bind-mounted into the container.
+This is where pleroma and postgres will store their data. + +#### Database (`SCRIPT_DEPLOY_POSTGRES`) + +Values: `true` / `false` + +By default pleroma-docker deploys a postgresql container and links it to pleroma’s container as a zero-config data store. +If you already have a postgres database or want to host it on a physically different machine, set this value to `false`. +Make sure to edit the `config :pleroma, Pleroma.Repo` variables when doing that. + +#### Reverse Proxy (`SCRIPT_USE_PROXY`) + +Values: `traefik` / `nginx` / `manual` + +Pleroma is usually run behind a reverse-proxy.
+Pleroma-docker gives you multiple options here. + +##### Traefik + +In traefik-mode we will generate a pleroma container with traefik-compatible labels. +These will be picked up at runtime to dynamically create a reverse-proxy configuration. +This should 'just work' if `watch=true` and `exposedByDefault=false` are set in the `[docker]` section of your `traefik.conf`. +SSL will also 'just work' once you add a matching `[[acme.domains]]` entry in there. + +##### NGINX + +In nginx-mode we will generate a bare nginx container that is linked to pleroma. +The nginx container is absolutely unmodified and expects to be configured by you. +The nginx file in [Pleroma's Repository](https://git.pleroma.social/pleroma/pleroma/blob/develop/installation/pleroma.nginx) is a good starting point. + +We will mount your configs like this: +```txt +custom.d/server.nginx -> /etc/nginx/nginx.conf +custom.d/vhost.nginx -> /etc/nginx/conf.d/pleroma.conf +``` + +To reach your pleroma container from inside nginx use `proxy_pass http://pleroma:4000;`. + +Set `SCRIPT_PORT_HTTP` and `SCRIPT_PORT_HTTPS` to the ports you want to listen on.
+Specify the ip to bind to in `SCRIPT_BIND_IP`. These values are required. + +The container only listens on `SCRIPT_PORT_HTTPS` if `SCRIPT_ENABLE_SSL` is `true`. + +##### Apache / httpd + +Just like nginx-mode this starts an unmodified apache server that expects to be configured by you.
+Again [Pleroma's Config](https://git.pleroma.social/pleroma/pleroma/blob/develop/installation/pleroma-apache.conf) is a good starting point. + +We will mount your configs like this: +``` +custom.d/server.httpd -> /usr/local/apache2/conf/httpd.conf +custom.d/vhost.httpd -> /usr/local/apache2/conf/extra/httpd-vhosts.conf +``` + +To reach your pleroma container from inside apache use `ProxyPass [loc] http://pleroma:4000/`. + +Again setting `SCRIPT_PORT_HTTP`, `SCRIPT_PORT_HTTPS` and `SCRIPT_BIND_IP` is required. + +The container only listens on `SCRIPT_PORT_HTTPS` if `SCRIPT_ENABLE_SSL` is `true`. + +##### Manual + +In manual mode we do not create any reverse proxy for you. +You'll have to figure something out on your own. + +If `SCRIPT_BIND_IN_MANUAL` is `true` we will forward `pleroma:4000` to `${SCRIPT_BIND_IP}:${SCRIPT_PORT_HTTP}`. + +**Pleroma's internal SSL implementation is currently not supported.** + +#### SSL (`SCRIPT_ENABLE_SSL`) + +Values: `true` / `false` + +If you want to use SSL with your Apache or NGINX containers you'll need a +certificate. Certificates need to be placed into `custom.d` and will be +bind-mounted into the server's container at runtime. + +We will mount your certs like this: +``` +custom.d/ssl.crt -> /ssl/ssl.crt +custom.d/ssl.key -> /ssl/ssl.key +``` + +You can reference them in Apache like this: +```apache + + SSLEngine on + SSLCertificateFile "/ssl/ssl.crt" + SSLCertificateKeyFile "/ssl/ssl.key" + +``` + +And in NGINX like this: +```nginx +listen 443 ssl; +ssl_certificate /ssl/ssl.crt; +ssl_certificate_key /ssl/ssl.key; +``` + +In traefik-mode and manual-mode these files and the `SCRIPT_ENABLE_SSL` value are ignored. From a8a1a6ed35bcb10b9576ef62003436737307189a Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 01:27:52 +0100 Subject: [PATCH 14/21] Update 'README.md' --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62d04fe..5203c50 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,19 @@ or guides from the community. A few are linked below. This list is not exhaustiv ## Docs +### Prerequisites + +- 1-2GB of free HDD space (yeah it sucks, Alpine Linux soontm) +- `m4` and `awk` in remotely recent versions +- `curl` or `wget` if you want smarter build caches and commands like `./pleroma mod` +- `jq` and `dialog` if you want to use `./pleroma mod` +- Bash 4.0+ (fancy scripting stuff) +- Docker 18.06.0+ and docker-compose 1.22.0-rc1+ (We need compose file format 3.7+ for `init:`) + ### Installation - Clone this repository -- Create a `config.exs` and `.env` +- Create a `config.exs` and `.env` file - Run `./pleroma build` and `./pleroma up` - Profit! From a02f0a4d67255671946427c3f3576d5fce11c6a6 Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 18:24:29 +0100 Subject: [PATCH 15/21] Implement smart build caches --- Dockerfile | 4 ++-- pleroma | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea9f50d..16c5150 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,8 +53,8 @@ WORKDIR /home/pleroma/pleroma # used version/branch/tag/commitish/... which originates in the script. # If the host doesn't have the required tool for "smart version detection" # we'll just use the current timestamp here which forces a rebuild every time. -ARG __BUST_CACHE -ENV __BUST_CACHE $__BUST_CACHE +ARG __CACHE_TAG +ENV __CACHE_TAG $__CACHE_TAG # Fetch changes, checkout ARG PLEROMA_VERSION diff --git a/pleroma b/pleroma index fa4f39d..2cb4d60 100755 --- a/pleroma +++ b/pleroma @@ -13,6 +13,7 @@ set -o pipefail readonly GITLAB_URI="https://git.pleroma.social" readonly PREFIX_API="api/v4/projects/pleroma%2Fpleroma/repository" +readonly ENDPOINT_REPO="pleroma/pleroma.git" readonly ENDPOINT_FILE="pleroma/pleroma/raw" readonly ENDPOINT_LIST="pleroma/pleroma/files" readonly ENDPOINT_TAG="$PREFIX_API/tags" @@ -110,7 +111,40 @@ request_file_content() { # $1: source ######################################################### action__build() { - docker_compose build --build-arg __BUST_CACHE="$(date +%s)" server + local cacheTag="" + + # Alternative 1: Get tags or branches from git (if installed) + if [[ -z "$cacheTag" ]] && has_command git; then + set +o pipefail + local resolvedHash + resolvedHash="$(git ls-remote $GITLAB_URI/$ENDPOINT_REPO | grep "/$PLEROMA_VERSION" | awk '{ print $1 }')" + set -o pipefail + + if [[ -n "$resolvedHash" ]]; then + cacheTag="$resolvedHash" + fi + fi + + # Alternative 2: Current time with date or awk + if [[ -z "$cacheTag" ]] && has_command date; then + cacheTag="$(date '+%s')" + fi + + if [[ -z "$cacheTag" ]] && has_command awk; then + cacheTag="$(awk 'BEGIN { srand(); print srand() }')" + fi + + # Last resort: Constant value, user will need to run `docker system prune` + if [[ -z "$cacheTag" ]]; then + echo "" + echo "Warning: Cannot set a reliably unique cache tag." + echo " You may be running outdated code!" + echo "" + + cacheTag="broken-host-env" + fi + + docker_compose build --build-arg __CACHE_TAG="$cacheTag" server } action__dump() { From 2acfbc88428a3be6322287ce2131297648a4a428 Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 19:03:52 +0100 Subject: [PATCH 16/21] Migrate to alpine linux wew that was easy --- Dockerfile | 25 +++++++++++-------------- README.md | 2 +- entrypoint.sh | 5 +++-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16c5150..0bd34a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,9 @@ -FROM debian:9-slim +FROM alpine:3.8 # Set up environment -ENV MIX_ENV=prod -ENV DEBIAN_FRONTEND=noninteractive ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 +ENV MIX_ENV=prod # Prepare mounts VOLUME /custom.d @@ -12,18 +11,16 @@ VOLUME /custom.d # Expose default pleroma port to host EXPOSE 4000 -# Get erlang, elixir, and dependencies +# Get dependencies RUN \ - apt-get update \ - && apt-get install -y --no-install-recommends apt-utils \ - && apt-get install -y --no-install-recommends git wget ca-certificates gnupg2 build-essential ruby \ + apk add --no-cache --virtual .tools \ + git curl \ \ - && wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb \ - && dpkg -i erlang-solutions_1.0_all.deb \ - && apt-get update \ - && apt-get install -y --no-install-recommends esl-erlang elixir \ + && apk add --no-cache --virtual .sdk \ + build-base \ \ - && rm -rf /var/lib/apt/lists/* + && apk add --no-cache --virtual .runtime \ + erlang erlang-runtime-tools erlang-xmerl elixir # Add entrypoint COPY ./entrypoint.sh / @@ -36,8 +33,8 @@ ARG DOCKER_GID=1000 ARG PLEROMA_UPLOADS_PATH=/uploads RUN \ - groupadd --gid ${DOCKER_GID} pleroma \ - && useradd -m -s /bin/bash --gid ${DOCKER_GID} --uid ${DOCKER_UID} pleroma \ + addgroup --gid ${DOCKER_GID} pleroma \ + && adduser -S -s /bin/ash -G pleroma -u ${DOCKER_UID} pleroma \ && mkdir -p /custom.d $PLEROMA_UPLOADS_PATH \ && chown -R pleroma:pleroma /custom.d $PLEROMA_UPLOADS_PATH diff --git a/README.md b/README.md index 5203c50..4556eca 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ or guides from the community. A few are linked below. This list is not exhaustiv ### Prerequisites -- 1-2GB of free HDD space (yeah it sucks, Alpine Linux soontm) +- 500mb of free HDD space - `m4` and `awk` in remotely recent versions - `curl` or `wget` if you want smarter build caches and commands like `./pleroma mod` - `jq` and `dialog` if you want to use `./pleroma mod` diff --git a/entrypoint.sh b/entrypoint.sh index 422e7fb..6abb603 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,9 +1,10 @@ -#!/bin/bash +#!/bin/ash +# shellcheck shell=dash set -e set -x -if [[ -n "$BUILDTIME" ]]; then +if [ -n "$BUILDTIME" ]; then mix local.rebar --force mix local.hex --force From a72cfa9b4e1c916dd98fa043b48e62fc7813411c Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 19:32:40 +0100 Subject: [PATCH 17/21] Wait for db during launch --- Dockerfile | 2 +- entrypoint.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bd34a5..2f0634c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ EXPOSE 4000 # Get dependencies RUN \ apk add --no-cache --virtual .tools \ - git curl \ + git curl postgresql-client \ \ && apk add --no-cache --virtual .sdk \ build-base \ diff --git a/entrypoint.sh b/entrypoint.sh index 6abb603..6f078d2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,10 +13,17 @@ if [ -n "$BUILDTIME" ]; then exit 0 fi +set +x +while ! pg_isready -U pleroma -d postgres://db:5432/pleroma -t 1; do + echo "[X] Database is starting up..." + sleep 1s +done +set -x + +# Recompile mix compile # Migrate db -mix ecto.create mix ecto.migrate # Off we go! From de32be4144b49bec690da736c6c45b948189c91b Mon Sep 17 00:00:00 2001 From: sn0w Date: Thu, 10 Jan 2019 19:49:47 +0100 Subject: [PATCH 18/21] Make custom.d work again --- Dockerfile | 2 +- entrypoint.sh | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f0634c..5e2ba04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ EXPOSE 4000 # Get dependencies RUN \ apk add --no-cache --virtual .tools \ - git curl postgresql-client \ + git curl rsync postgresql-client \ \ && apk add --no-cache --virtual .sdk \ build-base \ diff --git a/entrypoint.sh b/entrypoint.sh index 6f078d2..789dcaf 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,29 +2,32 @@ # shellcheck shell=dash set -e -set -x if [ -n "$BUILDTIME" ]; then + echo "#> Preparing compilation..." + mix local.rebar --force mix local.hex --force + echo "#> Compiling..." mix deps.get mix compile exit 0 fi -set +x -while ! pg_isready -U pleroma -d postgres://db:5432/pleroma -t 1; do - echo "[X] Database is starting up..." - sleep 1s -done -set -x +echo "#> Applying customizations and patches.." +rsync -av /custom.d/ /home/pleroma/pleroma/ -# Recompile +echo "#> Recompiling..." mix compile -# Migrate db +echo "#> Waiting until database is ready..." +while ! pg_isready -U pleroma -d postgres://db:5432/pleroma -t 1; do + sleep 1s +done + +echo "#> Upgrading database..." mix ecto.migrate -# Off we go! +echo "#> Liftoff!" exec mix phx.server From ffe592d34b440fc523309976cf7332f3ee65cfa0 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sat, 19 Jan 2019 23:52:27 +0100 Subject: [PATCH 19/21] Add config.exs to ignore --- .gitignore | 1 + config.exs | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 config.exs diff --git a/.gitignore b/.gitignore index 6bbc7ba..4e5fca7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ config.yml custom.d/ !custom.d/.gitkeep docker-compose.yml +config.exs # Created by https://www.gitignore.io/api/osx,linux,windows diff --git a/config.exs b/config.exs deleted file mode 100644 index 046b38e..0000000 --- a/config.exs +++ /dev/null @@ -1,14 +0,0 @@ -use Mix.Config - -# pleroma/pleroma/pleroma are the default credentials for the -# managed database container. "db" is the default interlinked hostname. -# You shouldn't need to change this unless you modifed .env -config :pleroma, Pleroma.Repo, - adapter: Ecto.Adapters.Postgres, - username: "pleroma", - password: "pleroma", - database: "pleroma", - hostname: "db", - pool_size: 10 - -# vvv Your awesome config options go here vvv From 4415a744070d8dc43fc6e0916181e195f4571d36 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 20 Jan 2019 01:45:25 +0100 Subject: [PATCH 20/21] Add some more examples to dist config --- config.dist.exs | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/config.dist.exs b/config.dist.exs index 046b38e..1e5a268 100644 --- a/config.dist.exs +++ b/config.dist.exs @@ -12,3 +12,75 @@ config :pleroma, Pleroma.Repo, pool_size: 10 # vvv Your awesome config options go here vvv + +### +# Here are some example values. +# Uncomment what you need or delete it all. +# +# Want to use the config generator instead? +# Try `./pleroma mix pleroma.instance gen` and then `./pleroma cp conf/generated_config.exs config.exs`. +# +# Need some inspiration? +# Take a look at https://git.pleroma.social/pleroma/pleroma/tree/develop/config +### + +# config :pleroma, Pleroma.Web.Endpoint, +# url: [host: "example.com", scheme: "https", port: 443], +# secret_key_base: "" + +# config :pleroma, :instance, +# name: "example instance", +# email: "example@example.com", +# limit: 5000, +# registrations_open: true, +# dedupe_media: false + +# config :pleroma, :media_proxy, +# enabled: false, +# redirect_on_failure: true +# base_url: "https://cache.example.com" + +# Configure web push notifications +# config :web_push_encryption, :vapid_details, +# subject: "mailto:example@example.com", +# public_key: "", +# private_key: "" + +# Enable Strict-Transport-Security once SSL is working: +# config :pleroma, :http_security, +# sts: true + +# Configure S3 support if desired. +# The public S3 endpoint is different depending on region and provider, +# consult your S3 provider's documentation for details on what to use. +# +# config :pleroma, Pleroma.Uploaders.S3, +# bucket: "some-bucket", +# public_endpoint: "https://s3.amazonaws.com" +# +# Configure S3 credentials: +# config :ex_aws, :s3, +# access_key_id: "xxxxxxxxxxxxx", +# secret_access_key: "yyyyyyyyyyyy", +# region: "us-east-1", +# scheme: "https://" +# +# For using third-party S3 clones like wasabi, also do: +# config :ex_aws, :s3, +# host: "s3.wasabisys.com" + +# Configure Openstack Swift support if desired. +# +# Many openstack deployments are different, so config is left very open with +# no assumptions made on which provider you're using. This should allow very +# wide support without needing separate handlers for OVH, Rackspace, etc. +# +# config :pleroma, Pleroma.Uploaders.Swift, +# container: "some-container", +# username: "api-username-yyyy", +# password: "api-key-xxxx", +# tenant_id: "", +# auth_url: "https://keystone-endpoint.provider.com", +# storage_url: "https://swift-endpoint.prodider.com/v1/AUTH_/", +# object_url: "https://cdn-endpoint.provider.com/" +# From 649528517313ac2d0097cc0ffa8250ff48f411b2 Mon Sep 17 00:00:00 2001 From: sn0w Date: Sun, 20 Jan 2019 02:13:51 +0100 Subject: [PATCH 21/21] Add cp command --- config.dist.exs | 2 +- pleroma | 49 ++++++++++++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/config.dist.exs b/config.dist.exs index 1e5a268..d5b477e 100644 --- a/config.dist.exs +++ b/config.dist.exs @@ -18,7 +18,7 @@ config :pleroma, Pleroma.Repo, # Uncomment what you need or delete it all. # # Want to use the config generator instead? -# Try `./pleroma mix pleroma.instance gen` and then `./pleroma cp conf/generated_config.exs config.exs`. +# Try `./pleroma mix pleroma.instance gen` and then `./pleroma cp /home/pleroma/pleroma/config/generated_config.exs config.exs`. # # Need some inspiration? # Take a look at https://git.pleroma.social/pleroma/pleroma/tree/develop/config diff --git a/pleroma b/pleroma index 2cb4d60..8631c47 100755 --- a/pleroma +++ b/pleroma @@ -270,6 +270,13 @@ action__mod() { fi } +action__cp() { + container="$(docker_compose ps -q server)" + + echo "$container:$1 -> $2" + docker cp "$container:$1" "$2" +} + ######################################################### # Help # ######################################################### @@ -282,36 +289,40 @@ Usage: $0 [action] [action-args...] Actions: - build (Re)build the pleroma container. + build (Re)build the pleroma container. - dump Dump the generated docker-compose.yml to stdout. + dump Dump the generated docker-compose.yml to stdout. - debug [bin] [args...] Launches a new pleroma container but uses \$bin instead of phx.server as entrypoint. - **Warning**: This is intended for debugging pleroma with tools like :debugger and :observer. - It thus forwards your X-Server into docker and temporarily fiddles with your xhost - access controls. If this is a security concern for you, please export NO_X_FORWARDING=1 - before launching a debugger session. + debug [bin] [args...] Launches a new pleroma container but uses \$bin instead of phx.server as entrypoint. + **Warning**: This is intended for debugging pleroma with tools like :debugger and :observer. + It thus forwards your X-Server into docker and temporarily fiddles with your xhost + access controls. If this is a security concern for you, please export NO_X_FORWARDING=1 + before launching a debugger session. - enter Spawn a shell inside the container for debugging/maintenance. - This command does not link to the postgres container. - If you need that use #debug instead. + enter Spawn a shell inside the container for debugging/maintenance. + This command does not link to the postgres container. + If you need that use #debug instead. - logs Show the current container logs. + logs Show the current container logs. - mix [task] [args...] Run a mix task without entering the container. + mix [task] [args...] Run a mix task without entering the container. - mod [file] Creates the file in custom.d and downloads the content from pleroma.social. - The download respects your \$PLEROMA_VERSION from .env. + mod [file] Creates the file in custom.d and downloads the content from pleroma.social. + The download respects your \$PLEROMA_VERSION from .env. - passthrough / p [...] Pass any custom command to docker-compose. + passthrough / p [...] Pass any custom command to docker-compose. - restart Executes #stop and #start respectively. + restart Executes #stop and #start respectively. - start / up Start pleroma and sibling services. + start / up Start pleroma and sibling services. - stop / down Stop pleroma and sibling services. + stop / down Stop pleroma and sibling services. - status / ps Show the current container status. + status / ps Show the current container status. + + copy / cp [source] [target] Copy a file from your pc to the pleroma container. + This operation only works in one direction. + For making permanent changes to the container use custom.d. Environment: DEBUG can be used to modify the loglevel.