diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/deploy-database.sh | 60 | ||||
| -rwxr-xr-x | scripts/fetch-player-dependencies.cr | 164 | ||||
| -rw-r--r-- | scripts/git/pre-commit | 17 | ||||
| -rwxr-xr-x | scripts/install-dependencies.sh | 174 |
4 files changed, 415 insertions, 0 deletions
diff --git a/scripts/deploy-database.sh b/scripts/deploy-database.sh new file mode 100755 index 00000000..fa24b8f0 --- /dev/null +++ b/scripts/deploy-database.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# +# Parameters +# + +interactive=true + +if [ "$1" = "--no-interactive" ]; then + interactive=false +fi + +# +# Enable and start Postgres +# + +sudo systemctl start postgresql.service +sudo systemctl enable postgresql.service + +# +# Create databse and user +# + +if [ "$interactive" = "true" ]; then + sudo -u postgres -- createuser -P kemal + sudo -u postgres -- createdb -O kemal invidious +else + # Generate a DB password + if [ -z "$POSTGRES_PASS" ]; then + echo "Generating database password" + POSTGRES_PASS=$(tr -dc 'A-Za-z0-9.;!?{[()]}\\/' < /dev/urandom | head -c16) + fi + + # hostname:port:database:username:password + echo "Writing .pgpass" + echo "127.0.0.1:*:invidious:kemal:${POSTGRES_PASS}" > "$HOME/.pgpass" + + sudo -u postgres -- psql -c "CREATE USER kemal WITH PASSWORD '$POSTGRES_PASS';" + sudo -u postgres -- psql -c "CREATE DATABASE invidious WITH OWNER kemal;" + sudo -u postgres -- psql -c "GRANT ALL ON DATABASE invidious TO kemal;" +fi + + +# +# Instructions for modification of pg_hba.conf +# + +if [ "$interactive" = "true" ]; then + echo + echo "-------------" + echo " NOTICE " + echo "-------------" + echo + echo "Make sure that your postgreSQL's pg_hba.conf file contains the follwong" + echo "lines before previous 'host' configurations:" + echo + echo "host invidious kemal 127.0.0.1/32 md5" + echo "host invidious kemal ::1/128 md5" + echo +fi diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr new file mode 100755 index 00000000..813e4ce4 --- /dev/null +++ b/scripts/fetch-player-dependencies.cr @@ -0,0 +1,164 @@ +require "http" +require "yaml" +require "digest/sha1" +require "option_parser" +require "colorize" + +# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html +minified = false +OptionParser.parse do |parser| + parser.banner = "Usage: Fetch VideoJS dependencies [arguments]" + parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { minified = true } + + parser.on("-h", "--help", "Show this help") do + puts parser + exit + end + + parser.invalid_option do |flag| + STDERR.puts "ERROR: #{flag} is not a valid option." + STDERR.puts parser + exit(1) + end +end + +required_dependencies = File.open("videojs-dependencies.yml") do |file| + YAML.parse(file).as_h +end + +def update_versions_yaml(required_dependencies, minified, dep_name) + File.open("assets/videojs/#{dep_name}/versions.yml", "w") do |io| + YAML.build(io) do |builder| + builder.mapping do + # Versions + builder.scalar "version" + builder.scalar "#{required_dependencies[dep_name]["version"]}" + + builder.scalar "minified" + builder.scalar minified + end + end + end +end + +# The first step is to check which dependencies we'll need to install. +# If the version we have requested in `videojs-dependencies.yml` is the +# same as what we've installed, we shouldn't do anything. Likewise, if it's +# different or the requested dependency just isn't present, then it needs to be +# installed. + +# Since we can't know when videojs-youtube-annotations is updated, we'll just always fetch +# a new copy each time. +dependencies_to_install = [] of String + +required_dependencies.keys.each do |dep| + dep = dep.to_s + path = "assets/videojs/#{dep}" + # Check for missing dependencies + if !Dir.exists?(path) + Dir.mkdir(path) + dependencies_to_install << dep + else + config = File.open("#{path}/versions.yml") do |file| + YAML.parse(file).as_h + end + + if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified + `rm -rf #{path}/*.js #{path}/*.css` + dependencies_to_install << dep + end + end +end + +# Now we begin the fun part of installing the dependencies. +# But first we'll setup a temp directory to store the plugins +tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install" +Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path + +channel = Channel(String | Exception).new + +dependencies_to_install.each do |dep| + spawn do + dep_name = dep + download_path = "#{tmp_dir_path}/#{dep}" + dest_path = "assets/videojs/#{dep}" + + HTTP::Client.get("https://registry.npmjs.org/#{dep}/-/#{dep}-#{required_dependencies[dep]["version"]}.tgz") do |response| + Dir.mkdir(download_path) + data = response.body_io.gets_to_end + File.write("#{download_path}/package.tgz", data) + + # https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908 + if `sha1sum #{download_path}/package.tgz`.split(" ")[0] != required_dependencies[dep]["shasum"] + raise Exception.new("Checksum for '#{dep}' failed") + end + end + + # Unless we install an external dependency, crystal provides no way of extracting a tarball. + # Thus we'll go ahead and call a system command. + `tar -vzxf '#{download_path}/package.tgz' -C '#{download_path}'` + raise "Extraction for #{dep} failed" if !$?.success? + + # Would use File.rename in the following steps but for some reason it just doesn't work here. + # Video.js itself is structured slightly differently + dep = "video" if dep == "video.js" + + # This dep nests everything under an additional JS or CSS folder + if dep == "silvermine-videojs-quality-selector" + js_path = "js/" + + # It also stores their quality selector as `quality-selector.css` + `mv #{download_path}/package/dist/css/quality-selector.css #{dest_path}/quality-selector.css` + else + js_path = "" + end + + # Would use File.rename but for some reason it just doesn't work here. + if minified && File.exists?("#{download_path}/package/dist/#{js_path}#{dep}.min.js") + `mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js` + else + `mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js` + end + + # Fetch CSS which isn't guaranteed to exist + # + # Also, video JS changes structure here once again... + dep = "video-js" if dep == "video" + + # VideoJS marker uses a dot on the CSS files. + dep = "videojs.markers" if dep == "videojs-markers" + + if File.exists?("#{download_path}/package/dist/#{dep}.css") + if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.css") + `mv #{download_path}/package/dist/#{dep}.min.css #{dest_path}/#{dep}.css` + else + `mv #{download_path}/package/dist/#{dep}.css #{dest_path}/#{dep}.css` + end + end + + # Update/create versions file for the dependency + update_versions_yaml(required_dependencies, minified, dep_name) + + channel.send(dep_name) + rescue ex + channel.send(ex) + end +end + +if dependencies_to_install.empty? + puts "#{"Player".colorize(:blue)} #{"dependencies".colorize(:green)} are satisfied" +else + puts "#{"Resolving".colorize(:green)} #{"player".colorize(:blue)} dependencies" + dependencies_to_install.size.times do + result = channel.receive + + if result.is_a? Exception + raise result + end + + puts "#{"Fetched".colorize(:green)} #{result.colorize(:blue)}" + end +end + +# Cleanup +`rm -rf #{tmp_dir_path}` diff --git a/scripts/git/pre-commit b/scripts/git/pre-commit new file mode 100644 index 00000000..4460b670 --- /dev/null +++ b/scripts/git/pre-commit @@ -0,0 +1,17 @@ +# Useful precomit hooks +# Please see https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks for instructions on installation. + +# Crystal linter +# This is a modified version of the pre-commit hook from the crystal repo. https://github.com/crystal-lang/crystal/blob/master/scripts/git/pre-commit +# Please refer to that if you'd like an version that doesn't automatically format staged files. +changed_cr_files=$(git diff --cached --name-only --diff-filter=ACM | grep '\.cr$') +if [ ! -z "$changed_cr_files" ]; then + if [ -x bin/crystal ]; then + # use bin/crystal wrapper when available to run local compiler build + bin/crystal tool format $changed_cr_files >&2 + else + crystal tool format $changed_cr_files >&2 + fi + + git add $changed_cr_files +fi diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh new file mode 100755 index 00000000..1e67bdaf --- /dev/null +++ b/scripts/install-dependencies.sh @@ -0,0 +1,174 @@ +#!/bin/sh +# +# Script that installs the various dependencies of invidious +# +# Dependencies: +# - crystal => Language in which Invidious is developed +# - postgres => Database server +# - git => required to clone Invidious +# - librsvg2-bin => For login captcha (provides 'rsvg-convert') +# +# - libssl-dev => Used by Crystal's SSL module (standard library) +# - libxml2-dev => Used by Crystal's XML module (standard library) +# - libyaml-dev => Used by Crystal's YAML module (standard library) +# - libgmp-dev => Used by Crystal's BigNumbers module (standard library) +# - libevent-dev => Used by crystal's internal scheduler (?) +# - libpcre3-dev => Used by Crystal's regex engine (?) +# +# - libsqlite3-dev => Used to open .db files from NewPipe exports +# - zlib1g-dev => TBD +# - libreadline-dev => TBD +# +# +# Tested on: +# - OpenSUSE Leap 15.3 + +# +# Load system details +# + +if [ -e /etc/os-release ]; then + . /etc/os-release +elif [ -e /usr/lib/os-release ]; then + . /usr/lib/os-release +else + echo "Unsupported Linux system" + exit 2 +fi + +# +# Some variables +# + +repo_base_url="https://download.opensuse.org/repositories/devel:/languages:/crystal/" +repo_end_url="devel:languages:crystal.repo" + +apt_gpg_key="/usr/share/keyrings/crystal.gpg" +apt_list_file="/etc/apt/sources.list.d/crystal.list" + +yum_repo_file="/etc/yum.repos.d/crystal.repo" + +# +# Major install functions +# + +make_repo_url() { + echo "${repo_base_url}/${1}/${repo_end_url}" +} + + +install_apt() { + repo="$1" + + echo "Adding Crystal repository" + + curl -fsSL "${repo_base_url}/${repo}/Release.key" \ + | gpg --dearmor \ + | sudo tee "${apt_gpg_key}" > /dev/null + + echo "deb [signed-by=${apt_gpg_key}] ${repo_base_url}/${repo}/ /" \ + | sudo tee "$apt_list_file" + + sudo apt-get update + + sudo apt-get install --yes --no-install-recommends \ + libssl-dev libxml2-dev libyaml-dev libgmp-dev libevent-dev \ + libpcre3-dev libreadline-dev libsqlite3-dev zlib1g-dev \ + crystal postgresql-13 git librsvg2-bin make +} + +install_yum() { + repo=$(make_repo_url "$1") + + echo "Adding Crystal repository" + + cat << END | sudo tee "${yum_repo_file}" > /dev/null +[crystal] +name=Crystal +type=rpm-md +baseurl=${repo}/ +gpgcheck=1 +gpgkey=${repo}/repodata/repomd.xml.key +enabled=1 +END + + sudo yum -y install \ + openssl-devel libxml2-devel libyaml-devel gmp-devel \ + readline-devel sqlite-devel \ + crystal postgresql postgresql-server git librsvg2-tools make +} + +install_pacman() { + # TODO: find an alternative to --no-confirm? + sudo pacman -S --no-confirm \ + base-devel librsvg postgresql crystal +} + +install_zypper() +{ + repo=$(make_repo_url "$1") + + echo "Adding Crystal repository" + sudo zypper --non-interactive addrepo -f "$repo" + + sudo zypper --non-interactive --gpg-auto-import-keys install --no-recommends \ + libopenssl-devel libxml2-devel libyaml-devel gmp-devel libevent-devel \ + pcre-devel readline-devel sqlite3-devel zlib-devel \ + crystal postgresql postgresql-server git rsvg-convert make +} + + +# +# System-specific logic +# + +case "$ID" in + archlinux) install_pacman;; + + centos) install_dnf "CentOS_${VERSION_ID}";; + + debian) + case "$VERSION_CODENAME" in + sid) install_apt "Debian_Unstable";; + bookworm) install_apt "Debian_Testing";; + *) install_apt "Debian_${VERSION_ID}";; + esac + ;; + + fedora) + if [ "$VERSION" == *"Prerelease"* ]; then + install_dnf "Fedora_Rawhide" + else + install_dnf "Fedora_${VERSION}" + fi + ;; + + opensuse-leap) install_zypper "openSUSE_Leap_${VERSION}";; + + opensuse-tumbleweed) install_zypper "openSUSE_Tumbleweed";; + + rhel) install_dnf "RHEL_${VERSION_ID}";; + + ubuntu) + # Small workaround for recently released 22.04 + case "$VERSION_ID" in + 22.04) install_apt "xUbuntu_21.04";; + *) install_apt "xUbuntu_${VERSION_ID}";; + esac + ;; + + *) + # Try to match on ID_LIKE instead + # Not guaranteed to 100% work + case "$ID_LIKE" in + archlinux) install_pacman;; + centos) install_dnf "CentOS_${VERSION_ID}";; + debian) install_apt "Debian_${VERSION_ID}";; + *) + echo "Error: distribution ${CODENAME} is not supported" + echo "Please install dependencies manually" + exit 2 + ;; + esac + ;; +esac |