Merge branch 'master' into unix-sockets

4 files changed, 415 insertions, 0 deletions

diff --git a/scripts/deploy-database.sh b/scripts/deploy-database.sh
new file mode 100755
index 00000000..fa24b8f0
--- /dev/null
+++ b/scripts/deploy-database.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+
+#
+# Parameters
+#
+
+interactive=true
+
+if [ "$1" = "--no-interactive" ]; then
+	interactive=false
+fi
+
+#
+# Enable and start Postgres
+#
+
+sudo systemctl start postgresql.service
+sudo systemctl enable postgresql.service
+
+#
+# Create databse and user
+#
+
+if [ "$interactive" = "true" ]; then
+	sudo -u postgres -- createuser -P kemal
+	sudo -u postgres -- createdb -O kemal invidious
+else
+	# Generate a DB password
+	if [ -z "$POSTGRES_PASS" ]; then
+		echo "Generating database password"
+		POSTGRES_PASS=$(tr -dc 'A-Za-z0-9.;!?{[()]}\\/' < /dev/urandom | head -c16)
+	fi
+
+	# hostname:port:database:username:password
+	echo "Writing .pgpass"
+	echo "127.0.0.1:*:invidious:kemal:${POSTGRES_PASS}" > "$HOME/.pgpass"
+
+	sudo -u postgres -- psql -c "CREATE USER kemal WITH PASSWORD '$POSTGRES_PASS';"
+	sudo -u postgres -- psql -c "CREATE DATABASE invidious WITH OWNER kemal;"
+	sudo -u postgres -- psql -c "GRANT ALL ON DATABASE invidious TO kemal;"
+fi
+
+
+#
+# Instructions for modification of pg_hba.conf
+#
+
+if [ "$interactive" = "true" ]; then
+	echo
+	echo "-------------"
+	echo "   NOTICE    "
+	echo "-------------"
+	echo
+	echo "Make sure that your postgreSQL's pg_hba.conf file contains the follwong"
+	echo "lines before previous 'host' configurations:"
+	echo
+	echo "host  invidious  kemal  127.0.0.1/32  md5"
+	echo "host  invidious  kemal  ::1/128       md5"
+	echo
+fi
diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr
new file mode 100755
index 00000000..813e4ce4
--- /dev/null
+++ b/scripts/fetch-player-dependencies.cr
@@ -0,0 +1,164 @@
+require "http"
+require "yaml"
+require "digest/sha1"
+require "option_parser"
+require "colorize"
+
+# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html
+minified = false
+OptionParser.parse do |parser|
+  parser.banner = "Usage: Fetch VideoJS dependencies [arguments]"
+  parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { minified = true }
+
+  parser.on("-h", "--help", "Show this help") do
+    puts parser
+    exit
+  end
+
+  parser.invalid_option do |flag|
+    STDERR.puts "ERROR: #{flag} is not a valid option."
+    STDERR.puts parser
+    exit(1)
+  end
+end
+
+required_dependencies = File.open("videojs-dependencies.yml") do |file|
+  YAML.parse(file).as_h
+end
+
+def update_versions_yaml(required_dependencies, minified, dep_name)
+  File.open("assets/videojs/#{dep_name}/versions.yml", "w") do |io|
+    YAML.build(io) do |builder|
+      builder.mapping do
+        # Versions
+        builder.scalar "version"
+        builder.scalar "#{required_dependencies[dep_name]["version"]}"
+
+        builder.scalar "minified"
+        builder.scalar minified
+      end
+    end
+  end
+end
+
+# The first step is to check which dependencies we'll need to install.
+# If the version we have requested in `videojs-dependencies.yml` is the
+# same as what we've installed, we shouldn't do anything. Likewise, if it's
+# different or the requested dependency just isn't present, then it needs to be
+# installed.
+
+# Since we can't know when videojs-youtube-annotations is updated, we'll just always fetch
+# a new copy each time.
+dependencies_to_install = [] of String
+
+required_dependencies.keys.each do |dep|
+  dep = dep.to_s
+  path = "assets/videojs/#{dep}"
+  # Check for missing dependencies
+  if !Dir.exists?(path)
+    Dir.mkdir(path)
+    dependencies_to_install << dep
+  else
+    config = File.open("#{path}/versions.yml") do |file|
+      YAML.parse(file).as_h
+    end
+
+    if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified
+      `rm -rf #{path}/*.js #{path}/*.css`
+      dependencies_to_install << dep
+    end
+  end
+end
+
+# Now we begin the fun part of installing the dependencies.
+# But first we'll setup a temp directory to store the plugins
+tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install"
+Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path
+
+channel = Channel(String | Exception).new
+
+dependencies_to_install.each do |dep|
+  spawn do
+    dep_name = dep
+    download_path = "#{tmp_dir_path}/#{dep}"
+    dest_path = "assets/videojs/#{dep}"
+
+    HTTP::Client.get("https://registry.npmjs.org/#{dep}/-/#{dep}-#{required_dependencies[dep]["version"]}.tgz") do |response|
+      Dir.mkdir(download_path)
+      data = response.body_io.gets_to_end
+      File.write("#{download_path}/package.tgz", data)
+
+      # https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908
+      if `sha1sum #{download_path}/package.tgz`.split(" ")[0] != required_dependencies[dep]["shasum"]
+        raise Exception.new("Checksum for '#{dep}' failed")
+      end
+    end
+
+    # Unless we install an external dependency, crystal provides no way of extracting a tarball.
+    # Thus we'll go ahead and call a system command.
+    `tar -vzxf '#{download_path}/package.tgz' -C '#{download_path}'`
+    raise "Extraction for #{dep} failed" if !$?.success?
+
+    # Would use File.rename in the following steps but for some reason it just doesn't work here.
+    # Video.js itself is structured slightly differently
+    dep = "video" if dep == "video.js"
+
+    # This dep nests everything under an additional JS or CSS folder
+    if dep == "silvermine-videojs-quality-selector"
+      js_path = "js/"
+
+      # It also stores their quality selector as `quality-selector.css`
+      `mv #{download_path}/package/dist/css/quality-selector.css #{dest_path}/quality-selector.css`
+    else
+      js_path = ""
+    end
+
+    # Would use File.rename but for some reason it just doesn't work here.
+    if minified && File.exists?("#{download_path}/package/dist/#{js_path}#{dep}.min.js")
+      `mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js`
+    else
+      `mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js`
+    end
+
+    # Fetch CSS which isn't guaranteed to exist
+    #
+    # Also, video JS changes structure here once again...
+    dep = "video-js" if dep == "video"
+
+    # VideoJS marker uses a dot on the CSS files.
+    dep = "videojs.markers" if dep == "videojs-markers"
+
+    if File.exists?("#{download_path}/package/dist/#{dep}.css")
+      if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.css")
+        `mv #{download_path}/package/dist/#{dep}.min.css #{dest_path}/#{dep}.css`
+      else
+        `mv #{download_path}/package/dist/#{dep}.css #{dest_path}/#{dep}.css`
+      end
+    end
+
+    # Update/create versions file for the dependency
+    update_versions_yaml(required_dependencies, minified, dep_name)
+
+    channel.send(dep_name)
+  rescue ex
+    channel.send(ex)
+  end
+end
+
+if dependencies_to_install.empty?
+  puts "#{"Player".colorize(:blue)} #{"dependencies".colorize(:green)} are satisfied"
+else
+  puts "#{"Resolving".colorize(:green)} #{"player".colorize(:blue)} dependencies"
+  dependencies_to_install.size.times do
+    result = channel.receive
+
+    if result.is_a? Exception
+      raise result
+    end
+
+    puts "#{"Fetched".colorize(:green)} #{result.colorize(:blue)}"
+  end
+end
+
+# Cleanup
+`rm -rf #{tmp_dir_path}`
diff --git a/scripts/git/pre-commit b/scripts/git/pre-commit
new file mode 100644
index 00000000..4460b670
--- /dev/null
+++ b/scripts/git/pre-commit
@@ -0,0 +1,17 @@
+# Useful precomit hooks
+# Please see https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks for instructions on installation.
+
+# Crystal linter
+# This is a modified version of the pre-commit hook from the crystal repo. https://github.com/crystal-lang/crystal/blob/master/scripts/git/pre-commit
+# Please refer to that if you'd like an version that doesn't automatically format staged files. 
+changed_cr_files=$(git diff --cached --name-only --diff-filter=ACM | grep '\.cr$')
+if [ ! -z "$changed_cr_files" ]; then
+  if [ -x bin/crystal ]; then
+    # use bin/crystal wrapper when available to run local compiler build
+    bin/crystal tool format $changed_cr_files >&2
+  else
+    crystal tool format $changed_cr_files >&2
+  fi
+
+  git add $changed_cr_files
+fi
diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh
new file mode 100755
index 00000000..1e67bdaf
--- /dev/null
+++ b/scripts/install-dependencies.sh
@@ -0,0 +1,174 @@
+#!/bin/sh
+#
+# Script that installs the various dependencies of invidious
+#
+# Dependencies:
+# - crystal       => Language in which Invidious is developed
+# - postgres      => Database server
+# - git           => required to clone Invidious
+# - librsvg2-bin  => For login captcha (provides 'rsvg-convert')
+#
+# - libssl-dev    => Used by Crystal's SSL module (standard library)
+# - libxml2-dev   => Used by Crystal's XML module (standard library)
+# - libyaml-dev   => Used by Crystal's YAML module (standard library)
+# - libgmp-dev    => Used by Crystal's BigNumbers module (standard library)
+# - libevent-dev  => Used by crystal's internal scheduler (?)
+# - libpcre3-dev  => Used by Crystal's regex engine (?)
+#
+# - libsqlite3-dev   => Used to open .db files from NewPipe exports
+# - zlib1g-dev       => TBD
+# - libreadline-dev  => TBD
+#
+#
+# Tested on:
+# - OpenSUSE Leap 15.3
+
+#
+# Load system details
+#
+
+if [ -e /etc/os-release ]; then
+	. /etc/os-release
+elif [ -e /usr/lib/os-release ]; then
+	. /usr/lib/os-release
+else
+	echo "Unsupported Linux system"
+	exit 2
+fi
+
+#
+# Some variables
+#
+
+repo_base_url="https://download.opensuse.org/repositories/devel:/languages:/crystal/"
+repo_end_url="devel:languages:crystal.repo"
+
+apt_gpg_key="/usr/share/keyrings/crystal.gpg"
+apt_list_file="/etc/apt/sources.list.d/crystal.list"
+
+yum_repo_file="/etc/yum.repos.d/crystal.repo"
+
+#
+# Major install functions
+#
+
+make_repo_url() {
+	echo "${repo_base_url}/${1}/${repo_end_url}"
+}
+
+
+install_apt() {
+	repo="$1"
+
+	echo "Adding Crystal repository"
+
+	curl -fsSL "${repo_base_url}/${repo}/Release.key" \
+		| gpg --dearmor \
+		| sudo tee "${apt_gpg_key}" > /dev/null
+
+	echo "deb [signed-by=${apt_gpg_key}] ${repo_base_url}/${repo}/ /" \
+		| sudo tee "$apt_list_file"
+
+	sudo apt-get update
+
+	sudo apt-get install --yes --no-install-recommends \
+		libssl-dev libxml2-dev libyaml-dev libgmp-dev libevent-dev \
+		libpcre3-dev libreadline-dev libsqlite3-dev zlib1g-dev \
+		crystal postgresql-13 git librsvg2-bin make
+}
+
+install_yum() {
+	repo=$(make_repo_url "$1")
+
+	echo "Adding Crystal repository"
+
+	cat << END | sudo tee "${yum_repo_file}" > /dev/null
+[crystal]
+name=Crystal
+type=rpm-md
+baseurl=${repo}/
+gpgcheck=1
+gpgkey=${repo}/repodata/repomd.xml.key
+enabled=1
+END
+
+	sudo yum -y install \
+		openssl-devel libxml2-devel libyaml-devel gmp-devel \
+		readline-devel sqlite-devel \
+		crystal postgresql postgresql-server git librsvg2-tools make
+}
+
+install_pacman() {
+	# TODO: find an alternative to --no-confirm?
+	sudo pacman -S --no-confirm \
+		base-devel librsvg postgresql crystal
+}
+
+install_zypper()
+{
+	repo=$(make_repo_url "$1")
+
+	echo "Adding Crystal repository"
+	sudo zypper --non-interactive addrepo -f "$repo"
+
+	sudo zypper --non-interactive --gpg-auto-import-keys install --no-recommends \
+		libopenssl-devel libxml2-devel libyaml-devel gmp-devel libevent-devel \
+		pcre-devel readline-devel sqlite3-devel zlib-devel \
+		crystal postgresql postgresql-server git rsvg-convert make
+}
+
+
+#
+# System-specific logic
+#
+
+case "$ID" in
+	archlinux) install_pacman;;
+
+	centos) install_dnf "CentOS_${VERSION_ID}";;
+
+	debian)
+		case "$VERSION_CODENAME" in
+			sid)      install_apt "Debian_Unstable";;
+			bookworm) install_apt "Debian_Testing";;
+			*)        install_apt "Debian_${VERSION_ID}";;
+		esac
+	;;
+
+	fedora)
+		if [ "$VERSION" == *"Prerelease"* ]; then
+			install_dnf "Fedora_Rawhide"
+		else
+			install_dnf "Fedora_${VERSION}"
+		fi
+	;;
+
+	opensuse-leap) install_zypper "openSUSE_Leap_${VERSION}";;
+
+	opensuse-tumbleweed) install_zypper "openSUSE_Tumbleweed";;
+
+	rhel) install_dnf "RHEL_${VERSION_ID}";;
+
+	ubuntu)
+		# Small workaround for recently released 22.04
+		case "$VERSION_ID" in
+			22.04) install_apt "xUbuntu_21.04";;
+			*)     install_apt "xUbuntu_${VERSION_ID}";;
+		esac
+	;;
+
+	*)
+		# Try to match on ID_LIKE instead
+		# Not guaranteed to 100% work
+		case "$ID_LIKE" in
+			archlinux) install_pacman;;
+			centos) install_dnf "CentOS_${VERSION_ID}";;
+			debian) install_apt "Debian_${VERSION_ID}";;
+			*)
+				echo "Error: distribution ${CODENAME} is not supported"
+				echo "Please install dependencies manually"
+				exit 2
+			;;
+		esac
+	;;
+esac