From 5971f03c0b3def67bcd719af06750d067b312270 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Mon, 12 Jun 2017 11:52:50 +0200 Subject: [PATCH 01/10] New upload endpoint implementation --- lib/asciinema/asciicasts.ex | 57 +++++++++++--- lib/asciinema/auth.ex | 11 +++ lib/asciinema/crypto.ex | 8 ++ lib/asciinema/users.ex | 57 ++++++++++++++ mix.exs | 4 + mix.lock | 6 +- spec/fixtures/5/asciicast.json | 17 +++++ test/asciinema/asciicasts_test.exs | 46 ++++++++++++ .../api/asciicast_controller_test.exs | 75 +++++++++++++++++++ test/support/data_case.ex | 55 ++++++++++++++ test/support/fixtures.ex | 14 ++-- web/controllers/api/asciicast_controller.ex | 44 +++++++++++ .../asciicast_animation_controller.ex | 4 +- web/controllers/asciicast_file_controller.ex | 4 +- web/controllers/asciicast_image_controller.ex | 4 +- web/models/api_token.ex | 28 +++++++ web/models/asciicast.ex | 45 +++++------ web/models/user.ex | 13 +++- web/router.ex | 4 + web/views/api/asciicast_view.ex | 11 +++ 20 files changed, 457 insertions(+), 50 deletions(-) create mode 100644 lib/asciinema/users.ex create mode 100644 spec/fixtures/5/asciicast.json create mode 100644 test/asciinema/asciicasts_test.exs create mode 100644 test/controllers/api/asciicast_controller_test.exs create mode 100644 test/support/data_case.ex create mode 100644 web/controllers/api/asciicast_controller.ex create mode 100644 web/models/api_token.ex create mode 100644 web/views/api/asciicast_view.ex diff --git a/lib/asciinema/asciicasts.ex b/lib/asciinema/asciicasts.ex index 2a117be..c8a2081 100644 --- a/lib/asciinema/asciicasts.ex +++ b/lib/asciinema/asciicasts.ex @@ -1,27 +1,64 @@ defmodule Asciinema.Asciicasts do + import Ecto.Query, warn: false alias Asciinema.{Repo, Asciicast, FileStore} + def get_asciicast!(id) when is_integer(id) do + Repo.get!(Asciicast, id) + end + def get_asciicast!(thing) when is_binary(thing) do + q = if String.length(thing) == 25 do + from a in Asciicast, where: a.secret_token == ^thing + else + case Integer.parse(thing) do + {id, ""} -> + from a in Asciicast, where: a.private == false and a.id == ^id + _ -> + from a in Asciicast, where: a.id == -1 # TODO fixme + end + end + + Repo.one!(q) + end + def create_asciicast(user, %Plug.Upload{path: path, filename: filename} = upload) do - asciicast = %Asciicast{user_id: user.id, file: filename} + asciicast = %Asciicast{user_id: user.id, + file: filename, + private: user.asciicasts_private_by_default} - with {:ok, json} <- File.read(path), - {:ok, attrs} <- Poison.decode(json), - {:ok, attrs} <- extract_attrs(attrs), - changeset = Asciicast.create_changeset(asciicast, attrs), - {:ok, %Asciicast{} = asciicast} <- Repo.insert(changeset) do - put_file(asciicast, upload) - {:ok, asciicast} - end + {_, result} = Repo.transaction(fn -> + with {:ok, json} <- File.read(path), + {:ok, attrs} <- Poison.decode(json), + {:ok, attrs} <- extract_attrs(attrs), + changeset = Asciicast.create_changeset(asciicast, attrs), + {:ok, %Asciicast{} = asciicast} <- Repo.insert(changeset) do + put_file(asciicast, upload) + # TODO: generate snapshot and poster + {:ok, asciicast} + else + {:error, :invalid} -> + {:error, :parse_error} + otherwise -> + otherwise + end + end) + + result end - defp extract_attrs(attrs) do + defp extract_attrs(%{"version" => 1} = attrs) do attrs = %{version: attrs["version"], duration: attrs["duration"], terminal_columns: attrs["width"], terminal_lines: attrs["height"], + terminal_type: get_in(attrs, ["env", "TERM"]), + command: attrs["command"], + shell: get_in(attrs, ["env", "SHELL"]), title: attrs["title"]} {:ok, attrs} end + defp extract_attrs(_attrs) do + {:error, :unknown_format} + end defp put_file(asciicast, %{path: tmp_file_path, content_type: content_type}) do file_store_path = Asciicast.json_store_path(asciicast) diff --git a/lib/asciinema/auth.ex b/lib/asciinema/auth.ex index a3f1fbc..6b7adf4 100644 --- a/lib/asciinema/auth.ex +++ b/lib/asciinema/auth.ex @@ -11,4 +11,15 @@ defmodule Asciinema.Auth do user = user_id && Repo.get(User, user_id) Conn.assign(conn, :current_user, user) end + + def get_basic_auth(conn) do + with ["Basic " <> auth] <- Conn.get_req_header(conn, "authorization"), + {:ok, username_password} <- Base.decode64(auth), + [username, password] <- String.split(username_password, ":") do + {username, password} + else + _ -> + nil + end + end end diff --git a/lib/asciinema/crypto.ex b/lib/asciinema/crypto.ex index 25a7f64..0530a82 100644 --- a/lib/asciinema/crypto.ex +++ b/lib/asciinema/crypto.ex @@ -2,4 +2,12 @@ defmodule Crypto do def md5(data) do Base.encode16(:erlang.md5(data), case: :lower) end + + def random_token(length) do + length + |> :crypto.strong_rand_bytes + |> Base.url_encode64 + |> String.replace(~r/[_=-]/, "") + |> binary_part(0, length) + end end diff --git a/lib/asciinema/users.ex b/lib/asciinema/users.ex new file mode 100644 index 0000000..1e2dae9 --- /dev/null +++ b/lib/asciinema/users.ex @@ -0,0 +1,57 @@ +defmodule Asciinema.Users do + import Ecto.Query, warn: false + alias Asciinema.{Repo, User, ApiToken} + + def authenticate(api_token) do + q = from u in User, + join: at in ApiToken, + on: at.user_id == u.id, + select: {u, at.revoked_at}, + where: at.token == ^api_token + + case Repo.one(q) do + nil -> + {:error, :token_not_found} + {%User{} = user, nil} -> + {:ok, user} + {%User{}, _} -> + {:error, :token_revoked} + end + end + + def get_user_with_api_token(username, api_token) do + case authenticate(api_token) do + {:ok, %User{} = user} -> + user + {:error, :token_revoked} -> + nil + {:error, :token_not_found} -> + create_user_with_api_token(username, api_token) + end + end + + def create_user_with_api_token(username, api_token) do + user_changeset = User.temporary_changeset(username) + + {_, result} = Repo.transaction(fn -> + with {:ok, %User{} = user} <- Repo.insert(user_changeset), + api_token_changeset = ApiToken.create_changeset(user, api_token), + {:ok, %ApiToken{}} <- Repo.insert(api_token_changeset) do + user + else + _otherwise -> + Repo.rollback(nil) + end + end) + + result + end + + def get_api_token!(token) do + Repo.get_by!(ApiToken, token: token) + end + + def revoke_api_token!(api_token) do + Repo.update!(ApiToken.revoke_changeset(api_token)) + end +end diff --git a/mix.exs b/mix.exs index e6bc849..372049c 100644 --- a/mix.exs +++ b/mix.exs @@ -33,6 +33,8 @@ defmodule Asciinema.Mixfile do :poolboy, :porcelain, :postgrex, + :timex, + :timex_ecto, ]] end @@ -61,6 +63,8 @@ defmodule Asciinema.Mixfile do {:poolboy, "~> 1.5"}, {:porcelain, "~> 2.0"}, {:postgrex, ">= 0.0.0"}, + {:timex, "~> 3.0"}, + {:timex_ecto, "~> 3.0"}, ] end diff --git a/mix.lock b/mix.lock index 8083361..2daa307 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{"briefly": {:hex, :briefly, "0.3.0", "16e6b76d2070ebc9cbd025fa85cf5dbaf52368c4bd896fb482b5a6b95a540c2f", [:mix], [], "hexpm"}, "bugsnag": {:hex, :bugsnag, "1.4.0", "fda8c3f550c93568b6e9ac615b1a9be0c1c4e06c7eb0ffb04a133dfaf1e01327", [:mix], [{:httpoison, "~> 0.9", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 1.5 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "certifi": {:hex, :certifi, "1.0.0", "1c787a85b1855ba354f0b8920392c19aa1d06b0ee1362f9141279620a5be2039", [:rebar3], [], "hexpm"}, + "combine": {:hex, :combine, "0.9.6", "8d1034a127d4cbf6924c8a5010d3534d958085575fa4d9b878f200d79ac78335", [:mix], [], "hexpm"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm"}, @@ -31,4 +32,7 @@ "porcelain": {:hex, :porcelain, "2.0.3", "2d77b17d1f21fed875b8c5ecba72a01533db2013bd2e5e62c6d286c029150fdc", [:mix], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.11.2", "139755c1359d3c5c6d6e8b1ea72556d39e2746f61c6ddfb442813c91f53487e8", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 1.0-rc", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}} + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}, + "timex": {:hex, :timex, "3.1.15", "94abaec8fef2436ced4d0e1b4ed50c8eaa5fb9138fc0699946ddee7abf5aaff2", [:mix], [{:combine, "~> 0.7", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, + "timex_ecto": {:hex, :timex_ecto, "3.0.5", "3ec6c25e10d2c0020958e5df64d2b5e690e441faa2c2259da8bc6bd3d7f39256", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.0", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, + "tzdata": {:hex, :tzdata, "0.5.12", "1c17b68692c6ba5b6ab15db3d64cc8baa0f182043d5ae9d4b6d35d70af76f67b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}} diff --git a/spec/fixtures/5/asciicast.json b/spec/fixtures/5/asciicast.json new file mode 100644 index 0000000..c37c327 --- /dev/null +++ b/spec/fixtures/5/asciicast.json @@ -0,0 +1,17 @@ +{ + "version": 5, + "width": 96, + "height": 26, + "duration": 11.146430015564, + "command": "/bin/bash", + "title": "bashing :)", + "env": { + "TERM": "screen-256color", + "SHELL": "/bin/zsh" + }, + "stdout": [ + [1.234567, "foo bar"], + [5.678987, "baz qux"], + [3.456789, "żółć jaźń"] + ] +} diff --git a/test/asciinema/asciicasts_test.exs b/test/asciinema/asciicasts_test.exs new file mode 100644 index 0000000..ecda8c4 --- /dev/null +++ b/test/asciinema/asciicasts_test.exs @@ -0,0 +1,46 @@ +defmodule Asciinema.AsciicastsTest do + use Asciinema.DataCase + alias Asciinema.Asciicasts + + describe "create_asciicast/2" do + test "json file, v1 format" do + user = fixture(:user) + upload = fixture(:upload, %{path: "1/asciicast.json"}) + + {:ok, asciicast} = Asciicasts.create_asciicast(user, upload) + + assert asciicast.version == 1 + assert asciicast.file == "asciicast.json" + assert asciicast.command == "/bin/bash" + assert asciicast.duration == 11.146430015564 + assert asciicast.shell == "/bin/zsh" + assert asciicast.terminal_type == "screen-256color" + assert asciicast.terminal_columns == 96 + assert asciicast.terminal_lines == 26 + assert asciicast.title == "bashing :)" + assert asciicast.uname == nil + # TODO assert asciicast.user_agent == "asciinema/1.0.0 gc/go1.3 jola-amd64" + end + + test "json file, v1 format (missing required data)" do + user = fixture(:user) + upload = fixture(:upload, %{path: "1/invalid.json"}) + + assert {:error, %Ecto.Changeset{}} = Asciicasts.create_asciicast(user, upload) + end + + test "json file, unsupported version number" do + user = fixture(:user) + upload = fixture(:upload, %{path: "5/asciicast.json"}) + + assert {:error, :unknown_format} = Asciicasts.create_asciicast(user, upload) + end + + test "non-json file" do + user = fixture(:user) + upload = fixture(:upload, %{path: "new-logo-bars.png"}) + + assert {:error, :parse_error} = Asciicasts.create_asciicast(user, upload) + end + end +end diff --git a/test/controllers/api/asciicast_controller_test.exs b/test/controllers/api/asciicast_controller_test.exs new file mode 100644 index 0000000..67323e7 --- /dev/null +++ b/test/controllers/api/asciicast_controller_test.exs @@ -0,0 +1,75 @@ +defmodule Asciinema.Api.AsciicastControllerTest do + use Asciinema.ConnCase + alias Asciinema.Users + + setup %{conn: conn} = context do + token = Map.get(context, :token, "9da34ff4-9bf7-45d4-aa88-98c933b15a3f") + + conn = if token do + put_req_header(conn, "authorization", "Basic " <> Base.encode64("test:" <> token)) + else + conn + end + + {:ok, conn: conn, token: token} + end + + @asciicast_url ~r|^http://localhost:4001/a/[a-zA-Z0-9]{25}| + + describe ".create" do + test "json file, v1 format", %{conn: conn} do + upload = fixture(:upload, %{path: "1/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert text_response(conn, 201) =~ @asciicast_url + assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url + end + + test "json file, v1 format (missing required data)", %{conn: conn} do + upload = fixture(:upload, %{path: "1/invalid.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert %{"errors" => _} = json_response(conn, 422) + end + + test "json file, unsupported version number", %{conn: conn} do + upload = fixture(:upload, %{path: "5/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert text_response(conn, 415) =~ ~r|not supported| + end + + test "non-json file", %{conn: conn} do + upload = fixture(:upload, %{path: "new-logo-bars.png"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert text_response(conn, 400) =~ ~r|valid asciicast| + end + + test "existing user (API token)", %{conn: conn, token: token} do + Users.create_user_with_api_token("test", token) + upload = fixture(:upload, %{path: "1/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert text_response(conn, 201) =~ @asciicast_url + assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url + end + + @tag token: nil + test "no authentication", %{conn: conn} do + upload = fixture(:upload, %{path: "1/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert response(conn, 401) + end + + test "authentication with revoked token", %{conn: conn, token: token} do + Users.get_user_with_api_token("test", token) # force registration of the token + token |> Users.get_api_token! |> Users.revoke_api_token! + upload = fixture(:upload, %{path: "1/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert response(conn, 401) + end + + @tag token: "invalid-lol" + test "authentication with invalid token", %{conn: conn} do + upload = fixture(:upload, %{path: "1/asciicast.json"}) + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} + assert response(conn, 401) + end + end +end diff --git a/test/support/data_case.ex b/test/support/data_case.ex new file mode 100644 index 0000000..6ed7b21 --- /dev/null +++ b/test/support/data_case.ex @@ -0,0 +1,55 @@ +defmodule Asciinema.DataCase do + @moduledoc """ + This module defines the setup for tests requiring + access to the application's data layer. + + You may define functions here to be used as helpers in + your tests. + + Finally, if the test case interacts with the database, + it cannot be async. For this reason, every test runs + inside a transaction which is reset at the beginning + of the test unless the test case is marked as async. + """ + + use ExUnit.CaseTemplate + + using do + quote do + alias Asciinema.Repo + + import Ecto + import Ecto.Changeset + import Ecto.Query + import Asciinema.DataCase + + import Asciinema.Fixtures + end + end + + setup tags do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(Asciinema.Repo) + + unless tags[:async] do + Ecto.Adapters.SQL.Sandbox.mode(Asciinema.Repo, {:shared, self()}) + end + + :ok + end + + @doc """ + A helper that transform changeset errors to a map of messages. + + assert {:error, changeset} = Accounts.create_user(%{password: "short"}) + assert "password is too short" in errors_on(changeset).password + assert %{password: ["password is too short"]} = errors_on(changeset) + + """ + def errors_on(changeset) do + Ecto.Changeset.traverse_errors(changeset, fn {message, opts} -> + Enum.reduce(opts, message, fn {key, value}, acc -> + String.replace(acc, "%{#{key}}", to_string(value)) + end) + end) + end +end diff --git a/test/support/fixtures.ex b/test/support/fixtures.ex index 6fd3e60..88c9d43 100644 --- a/test/support/fixtures.ex +++ b/test/support/fixtures.ex @@ -1,19 +1,23 @@ defmodule Asciinema.Fixtures do alias Asciinema.{Repo, Asciicasts, User} - def fixture(:upload) do - %Plug.Upload{path: "resources/welcome.json", - filename: "welcome.json", + def fixture(what, attrs \\ %{}) + + def fixture(:upload, attrs) do + path = Map.get(attrs, :path) || "1/asciicast.json" + filename = Path.basename(path) + %Plug.Upload{path: "spec/fixtures/#{path}", + filename: filename, content_type: "application/json"} end - def fixture(:user) do + def fixture(:user, _attrs) do attrs = %{username: "test", auth_token: "authy-auth-auth"} Repo.insert!(User.changeset(%User{}, attrs)) end - def fixture(:asciicast) do + def fixture(:asciicast, _attrs) do user = fixture(:user) upload = fixture(:upload) {:ok, asciicast} = Asciicasts.create_asciicast(user, upload) diff --git a/web/controllers/api/asciicast_controller.ex b/web/controllers/api/asciicast_controller.ex new file mode 100644 index 0000000..c7d7059 --- /dev/null +++ b/web/controllers/api/asciicast_controller.ex @@ -0,0 +1,44 @@ +defmodule Asciinema.Api.AsciicastController do + use Asciinema.Web, :controller + import Asciinema.Auth, only: [get_basic_auth: 1] + alias Asciinema.{Asciicasts, Users, User} + + plug :authenticate + + def create(conn, %{"asciicast" => %Plug.Upload{} = upload}) do + user = conn.assigns.current_user + + case Asciicasts.create_asciicast(user, upload) do + {:ok, asciicast} -> + url = asciicast_url(conn, :show, asciicast) + conn + |> put_status(:created) + |> put_resp_header("location", url) + |> text(url) + {:error, :parse_error} -> + conn + |> put_status(:bad_request) + |> text("This doesn't look like a valid asciicast file") + {:error, :unknown_format} -> + conn + |> put_status(:unsupported_media_type) + |> text("Format not supported") + {:error, %Ecto.Changeset{} = changeset} -> + conn + |> put_status(:unprocessable_entity) + |> render("error.json", changeset: changeset) + end + end + + defp authenticate(conn, _opts) do + with {username, api_token} <- get_basic_auth(conn), + %User{} = user <- Users.get_user_with_api_token(username, api_token) do + assign(conn, :current_user, user) + else + _otherwise -> + conn + |> send_resp(401, "Invalid or revoked recorder token") + |> halt() + end + end +end diff --git a/web/controllers/asciicast_animation_controller.ex b/web/controllers/asciicast_animation_controller.ex index cbbdcaf..f7dfa2a 100644 --- a/web/controllers/asciicast_animation_controller.ex +++ b/web/controllers/asciicast_animation_controller.ex @@ -1,9 +1,9 @@ defmodule Asciinema.AsciicastAnimationController do use Asciinema.Web, :controller - alias Asciinema.{Repo, Asciicast} + alias Asciinema.Asciicasts def show(conn, %{"id" => id}) do - asciicast = Repo.one!(Asciicast.by_id_or_secret_token(id)) + asciicast = Asciicasts.get_asciicast!(id) conn |> put_layout("simple.html") diff --git a/web/controllers/asciicast_file_controller.ex b/web/controllers/asciicast_file_controller.ex index e593cfe..c71e02b 100644 --- a/web/controllers/asciicast_file_controller.ex +++ b/web/controllers/asciicast_file_controller.ex @@ -1,9 +1,9 @@ defmodule Asciinema.AsciicastFileController do use Asciinema.Web, :controller - alias Asciinema.{Repo, Asciicast} + alias Asciinema.{Asciicasts, Asciicast} def show(conn, %{"id" => id} = params) do - asciicast = Repo.one!(Asciicast.by_id_or_secret_token(id)) + asciicast = Asciicasts.get_asciicast!(id) path = Asciicast.json_store_path(asciicast) filename = download_filename(asciicast, params) diff --git a/web/controllers/asciicast_image_controller.ex b/web/controllers/asciicast_image_controller.ex index 4be4e5e..8402a84 100644 --- a/web/controllers/asciicast_image_controller.ex +++ b/web/controllers/asciicast_image_controller.ex @@ -1,12 +1,12 @@ defmodule Asciinema.AsciicastImageController do use Asciinema.Web, :controller - alias Asciinema.{Repo, Asciicast, PngGenerator} + alias Asciinema.{Asciicasts, Asciicast, PngGenerator} alias Plug.MIME @max_age 604800 # 7 days def show(conn, %{"id" => id} = _params) do - asciicast = Repo.one!(Asciicast.by_id_or_secret_token(id)) + asciicast = Asciicasts.get_asciicast!(id) user = Repo.preload(asciicast, :user).user png_params = Asciicast.png_params(asciicast, user) diff --git a/web/models/api_token.ex b/web/models/api_token.ex new file mode 100644 index 0000000..e0c0906 --- /dev/null +++ b/web/models/api_token.ex @@ -0,0 +1,28 @@ +defmodule Asciinema.ApiToken do + use Asciinema.Web, :model + alias Asciinema.{ApiToken, User} + + schema "api_tokens" do + field :token, :string + field :revoked_at, Timex.Ecto.DateTime + + timestamps(inserted_at: :created_at) + + belongs_to :user, Asciinema.User + end + + @uuid4 ~r/\A[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\z/ + + def create_changeset(%User{id: user_id}, token) do + %ApiToken{user_id: user_id} + |> change(%{token: token}) + |> validate_format(:token, @uuid4) + end + + def revoke_changeset(%ApiToken{revoked_at: nil} = api_token) do + change(api_token, %{revoked_at: Timex.now()}) + end + def revoke_changeset(%ApiToken{} = api_token) do + change(api_token) + end +end diff --git a/web/models/asciicast.ex b/web/models/asciicast.ex index 1bc3d7d..6123a92 100644 --- a/web/models/asciicast.ex +++ b/web/models/asciicast.ex @@ -11,6 +11,7 @@ defmodule Asciinema.Asciicast do field :file, :string field :terminal_columns, :integer field :terminal_lines, :integer + field :terminal_type, :string field :stdout_data, :string field :stdout_timing, :string field :stdout_frames, :string @@ -20,48 +21,38 @@ defmodule Asciinema.Asciicast do field :title, :string field :theme_name, :string field :snapshot_at, :float + field :command, :string + field :shell, :string + field :uname, :string timestamps(inserted_at: :created_at) belongs_to :user, User end - def changeset(struct, attrs \\ %{}) do - struct - |> cast(attrs, [:title]) + defimpl Phoenix.Param, for: Asciicast do + def to_param(%Asciicast{private: false, id: id}) do + Integer.to_string(id) + end + def to_param(%Asciicast{private: true, secret_token: secret_token}) do + secret_token + end end def create_changeset(struct, attrs) do struct - |> changeset(attrs) - |> cast(attrs, [:user_id, :version, :file, :duration, :terminal_columns, :terminal_lines]) + |> cast(attrs, [:version, :file, :duration, :terminal_columns, :terminal_lines, :terminal_type, :command, :shell, :title, :uname]) + |> validate_required([:user_id, :private, :version, :duration, :terminal_columns, :terminal_lines]) |> generate_secret_token - |> validate_required([:user_id, :version, :duration, :terminal_columns, :terminal_lines, :secret_token]) end - defp generate_secret_token(changeset) do - put_change(changeset, :secret_token, random_token(25)) - end - - defp random_token(length) do - length - |> :crypto.strong_rand_bytes - |> Base.url_encode64 - |> String.replace(~r/[_=-]/, "") - |> binary_part(0, length) + def update_changeset(struct, attrs) do + struct + |> cast(attrs, [:title, :theme_name, :snapshot_at]) end - def by_id_or_secret_token(thing) do - if String.length(thing) == 25 do - from a in __MODULE__, where: a.secret_token == ^thing - else - case Integer.parse(thing) do - {id, ""} -> - from a in __MODULE__, where: a.private == false and a.id == ^id - :error -> - from a in __MODULE__, where: a.id == -1 # TODO fixme - end - end + defp generate_secret_token(changeset) do + put_change(changeset, :secret_token, Crypto.random_token(25)) end def json_store_path(%__MODULE__{id: id, file: file}) when is_binary(file) do diff --git a/web/models/user.ex b/web/models/user.ex index d15f604..270af52 100644 --- a/web/models/user.ex +++ b/web/models/user.ex @@ -1,5 +1,6 @@ defmodule Asciinema.User do use Asciinema.Web, :model + alias Asciinema.User schema "users" do field :username, :string @@ -20,7 +21,17 @@ defmodule Asciinema.User do """ def changeset(struct, params \\ %{}) do struct - |> cast(params, [:email, :name, :username, :temporary_username, :auth_token, :theme_name, :asciicasts_private_by_default]) + |> cast(params, [:email, :name, :username, :auth_token, :theme_name, :asciicasts_private_by_default]) |> validate_required([:auth_token]) end + + def temporary_changeset(temporary_username) do + %User{} + |> change(%{temporary_username: temporary_username}) + |> generate_auth_token + end + + defp generate_auth_token(changeset) do + put_change(changeset, :auth_token, Crypto.random_token(20)) + end end diff --git a/web/router.ex b/web/router.ex index d3a765e..f248394 100644 --- a/web/router.ex +++ b/web/router.ex @@ -52,6 +52,10 @@ defmodule Asciinema.Router do get "/docs/:topic", DocController, :show end + scope "/api", Asciinema.Api, as: :api do + post "/asciicasts", AsciicastController, :create + end + # Other scopes may use custom stacks. # scope "/api", Asciinema do # pipe_through :api diff --git a/web/views/api/asciicast_view.ex b/web/views/api/asciicast_view.ex new file mode 100644 index 0000000..7286e0b --- /dev/null +++ b/web/views/api/asciicast_view.ex @@ -0,0 +1,11 @@ +defmodule Asciinema.Api.AsciicastView do + use Asciinema.Web, :view + + def translate_errors(changeset) do + Ecto.Changeset.traverse_errors(changeset, &translate_error/1) + end + + def render("error.json", %{changeset: changeset}) do + %{errors: translate_errors(changeset)} + end +end From f34b9707f37646406348f440354b174f0ee81777 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Mon, 12 Jun 2017 12:08:28 +0200 Subject: [PATCH 02/10] Reformat code --- lib/asciinema/auth.ex | 3 +-- lib/asciinema/users.ex | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/asciinema/auth.ex b/lib/asciinema/auth.ex index 6b7adf4..b1ddae3 100644 --- a/lib/asciinema/auth.ex +++ b/lib/asciinema/auth.ex @@ -18,8 +18,7 @@ defmodule Asciinema.Auth do [username, password] <- String.split(username_password, ":") do {username, password} else - _ -> - nil + _ -> nil end end end diff --git a/lib/asciinema/users.ex b/lib/asciinema/users.ex index 1e2dae9..bfaab45 100644 --- a/lib/asciinema/users.ex +++ b/lib/asciinema/users.ex @@ -39,8 +39,7 @@ defmodule Asciinema.Users do {:ok, %ApiToken{}} <- Repo.insert(api_token_changeset) do user else - _otherwise -> - Repo.rollback(nil) + _otherwise -> Repo.rollback(nil) end end) @@ -52,6 +51,8 @@ defmodule Asciinema.Users do end def revoke_api_token!(api_token) do - Repo.update!(ApiToken.revoke_changeset(api_token)) + api_token + |> ApiToken.revoke_changeset + |> Repo.update! end end From 515d4863096e3518e59ee4deb6d97b2f0059715d Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Thu, 15 Jun 2017 22:25:04 +0200 Subject: [PATCH 03/10] Support client version <= 0.9.7 in new upload endpoint --- lib/asciinema/asciicasts.ex | 41 ++++++++++++++--- lib/asciinema/auth.ex | 8 ++++ test/asciinema/asciicasts_test.exs | 38 +++++++++++++++- .../api/asciicast_controller_test.exs | 14 ++++++ web/controllers/api/asciicast_controller.ex | 45 ++++++++++++++++++- web/models/asciicast.ex | 25 +++++++++-- 6 files changed, 158 insertions(+), 13 deletions(-) diff --git a/lib/asciinema/asciicasts.ex b/lib/asciinema/asciicasts.ex index c8a2081..4014f89 100644 --- a/lib/asciinema/asciicasts.ex +++ b/lib/asciinema/asciicasts.ex @@ -20,8 +20,11 @@ defmodule Asciinema.Asciicasts do Repo.one!(q) end - def create_asciicast(user, %Plug.Upload{path: path, filename: filename} = upload) do + def create_asciicast(user, params, user_agent \\ nil) + + def create_asciicast(user, %Plug.Upload{path: path, filename: filename} = upload, user_agent) do asciicast = %Asciicast{user_id: user.id, + user_agent: user_agent, file: filename, private: user.asciicasts_private_by_default} @@ -31,8 +34,8 @@ defmodule Asciinema.Asciicasts do {:ok, attrs} <- extract_attrs(attrs), changeset = Asciicast.create_changeset(asciicast, attrs), {:ok, %Asciicast{} = asciicast} <- Repo.insert(changeset) do - put_file(asciicast, upload) - # TODO: generate snapshot and poster + save_file(asciicast, :file, upload) + generate_poster(asciicast) {:ok, asciicast} else {:error, :invalid} -> @@ -45,6 +48,30 @@ defmodule Asciinema.Asciicasts do result end + def create_asciicast(user, %{"meta" => attrs, + "stdout" => %Plug.Upload{filename: d_filename} = data, + "stdout_timing" => %Plug.Upload{filename: t_filename} = timing}, _user_agent) do + attrs = Map.put(attrs, "version", 0) + asciicast = %Asciicast{user_id: user.id, + stdout_data: d_filename, + stdout_timing: t_filename, + private: user.asciicasts_private_by_default} + + changeset = Asciicast.create_changeset(asciicast, attrs) + {_, result} = Repo.transaction(fn -> + with {:ok, %Asciicast{} = asciicast} <- Repo.insert(changeset) do + save_file(asciicast, :stdout_data, data) + save_file(asciicast, :stdout_timing, timing) + generate_poster(asciicast) + {:ok, asciicast} + else + otherwise -> otherwise + end + end) + + result + end + defp extract_attrs(%{"version" => 1} = attrs) do attrs = %{version: attrs["version"], duration: attrs["duration"], @@ -60,8 +87,12 @@ defmodule Asciinema.Asciicasts do {:error, :unknown_format} end - defp put_file(asciicast, %{path: tmp_file_path, content_type: content_type}) do - file_store_path = Asciicast.json_store_path(asciicast) + defp save_file(asciicast, type, %{path: tmp_file_path, content_type: content_type}) do + file_store_path = Asciicast.file_store_path(asciicast, type) :ok = FileStore.put_file(file_store_path, tmp_file_path, content_type) end + + defp generate_poster(_asciicast) do + # TODO + end end diff --git a/lib/asciinema/auth.ex b/lib/asciinema/auth.ex index b1ddae3..8d74605 100644 --- a/lib/asciinema/auth.ex +++ b/lib/asciinema/auth.ex @@ -21,4 +21,12 @@ defmodule Asciinema.Auth do _ -> nil end end + + def put_basic_auth(conn, nil, nil) do + conn + end + def put_basic_auth(conn, username, password) do + auth = Base.encode64("#{username}:#{password}") + Conn.put_req_header(conn, "authorization", "Basic " <> auth) + end end diff --git a/test/asciinema/asciicasts_test.exs b/test/asciinema/asciicasts_test.exs index ecda8c4..f964f56 100644 --- a/test/asciinema/asciicasts_test.exs +++ b/test/asciinema/asciicasts_test.exs @@ -3,14 +3,48 @@ defmodule Asciinema.AsciicastsTest do alias Asciinema.Asciicasts describe "create_asciicast/2" do + test "json file, v0 format, <= v0.9.7 client" do + user = fixture(:user) + params = %{"meta" => %{"command" => "/bin/bash", + "duration" => 11.146430015564, + "shell" => "/bin/zsh", + "terminal_columns" => 96, + "terminal_lines" => 26, + "terminal_type" => "screen-256color", + "title" => "bashing :)", + "uname" => "Linux 3.9.9-302.fc19.x86_64 #1 SMP Sat Jul 6 13:41:07 UTC 2013 x86_64"}, + "stdout" => fixture(:upload, %{path: "0.9.7/stdout", + content_type: "application/octet-stream"}), + "stdout_timing" => fixture(:upload, %{path: "0.9.7/stdout.time", + content_type: "application/octet-stream"})} + + {:ok, asciicast} = Asciicasts.create_asciicast(user, params, "a/user/agent") + + assert asciicast.version == 0 + assert asciicast.file == nil + assert asciicast.stdout_data == "stdout" + assert asciicast.stdout_timing == "stdout.time" + assert asciicast.command == "/bin/bash" + assert asciicast.duration == 11.146430015564 + assert asciicast.shell == "/bin/zsh" + assert asciicast.terminal_type == "screen-256color" + assert asciicast.terminal_columns == 96 + assert asciicast.terminal_lines == 26 + assert asciicast.title == "bashing :)" + assert asciicast.uname == "Linux 3.9.9-302.fc19.x86_64 #1 SMP Sat Jul 6 13:41:07 UTC 2013 x86_64" + assert asciicast.user_agent == nil + end + test "json file, v1 format" do user = fixture(:user) upload = fixture(:upload, %{path: "1/asciicast.json"}) - {:ok, asciicast} = Asciicasts.create_asciicast(user, upload) + {:ok, asciicast} = Asciicasts.create_asciicast(user, upload, "a/user/agent") assert asciicast.version == 1 assert asciicast.file == "asciicast.json" + assert asciicast.stdout_data == nil + assert asciicast.stdout_timing == nil assert asciicast.command == "/bin/bash" assert asciicast.duration == 11.146430015564 assert asciicast.shell == "/bin/zsh" @@ -19,7 +53,7 @@ defmodule Asciinema.AsciicastsTest do assert asciicast.terminal_lines == 26 assert asciicast.title == "bashing :)" assert asciicast.uname == nil - # TODO assert asciicast.user_agent == "asciinema/1.0.0 gc/go1.3 jola-amd64" + assert asciicast.user_agent == "a/user/agent" end test "json file, v1 format (missing required data)" do diff --git a/test/controllers/api/asciicast_controller_test.exs b/test/controllers/api/asciicast_controller_test.exs index 67323e7..742d1f3 100644 --- a/test/controllers/api/asciicast_controller_test.exs +++ b/test/controllers/api/asciicast_controller_test.exs @@ -17,6 +17,20 @@ defmodule Asciinema.Api.AsciicastControllerTest do @asciicast_url ~r|^http://localhost:4001/a/[a-zA-Z0-9]{25}| describe ".create" do + @tag token: nil + test "separate files (pre-v1 params), v0.9.7 client", %{conn: conn} do + asciicast = %{"meta" => fixture(:upload, %{path: "0.9.7/meta.json", + content_type: "application/json"}), + "stdout" => fixture(:upload, %{path: "0.9.7/stdout", + content_type: "application/octet-stream"}), + "stdout_timing" => fixture(:upload, %{path: "0.9.7/stdout.time", + content_type: "application/octet-stream"})} + + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => asciicast} + assert text_response(conn, 201) =~ @asciicast_url + assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url + end + test "json file, v1 format", %{conn: conn} do upload = fixture(:upload, %{path: "1/asciicast.json"}) conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} diff --git a/web/controllers/api/asciicast_controller.ex b/web/controllers/api/asciicast_controller.ex index c7d7059..b1958bc 100644 --- a/web/controllers/api/asciicast_controller.ex +++ b/web/controllers/api/asciicast_controller.ex @@ -1,14 +1,25 @@ defmodule Asciinema.Api.AsciicastController do use Asciinema.Web, :controller - import Asciinema.Auth, only: [get_basic_auth: 1] + import Asciinema.Auth, only: [get_basic_auth: 1, put_basic_auth: 3] alias Asciinema.{Asciicasts, Users, User} + plug :parse_v0_params plug :authenticate def create(conn, %{"asciicast" => %Plug.Upload{} = upload}) do + do_create(conn, upload) + end + def create(conn, %{"asciicast" => %{"meta" => %{}, + "stdout" => %Plug.Upload{}, + "stdout_timing" => %Plug.Upload{}} = asciicast_params}) do + do_create(conn, asciicast_params) + end + + defp do_create(conn, params) do user = conn.assigns.current_user + user_agent = conn |> get_req_header("user-agent") |> List.first - case Asciicasts.create_asciicast(user, upload) do + case Asciicasts.create_asciicast(user, params, user_agent) do {:ok, asciicast} -> url = asciicast_url(conn, :show, asciicast) conn @@ -30,6 +41,36 @@ defmodule Asciinema.Api.AsciicastController do end end + defp parse_v0_params(%Plug.Conn{params: %{"asciicast" => %{"meta" => %Plug.Upload{path: meta_path}}}} = conn, _) do + with {:ok, json} <- File.read(meta_path), + {:ok, attrs} <- Poison.decode(json), + {:ok, meta} <- extract_v0_attrs(attrs) do + conn + |> put_param(["asciicast", "meta"], meta) + |> put_basic_auth(attrs["username"], attrs["user_token"]) + else + {:error, :invalid} -> + send_resp(conn, 400, "") + end + end + defp parse_v0_params(conn, _), do: conn + + defp put_param(%Plug.Conn{params: params} = conn, path, value) do + params = put_in(params, path, value) + %{conn | params: params} + end + + defp extract_v0_attrs(attrs) do + attrs = Map.merge( + Map.take(attrs, ["command", "duration", "shell", "title", "uname"]), + %{"terminal_columns" => get_in(attrs, ["term", "columns"]), + "terminal_lines" => get_in(attrs, ["term", "lines"]), + "terminal_type" => get_in(attrs, ["term", "type"])} + ) + + {:ok, attrs} + end + defp authenticate(conn, _opts) do with {username, api_token} <- get_basic_auth(conn), %User{} = user <- Users.get_user_with_api_token(username, api_token) do diff --git a/web/models/asciicast.ex b/web/models/asciicast.ex index 6123a92..5e3465b 100644 --- a/web/models/asciicast.ex +++ b/web/models/asciicast.ex @@ -24,6 +24,7 @@ defmodule Asciinema.Asciicast do field :command, :string field :shell, :string field :uname, :string + field :user_agent, :string timestamps(inserted_at: :created_at) @@ -55,11 +56,27 @@ defmodule Asciinema.Asciicast do put_change(changeset, :secret_token, Crypto.random_token(25)) end - def json_store_path(%__MODULE__{id: id, file: file}) when is_binary(file) do - "asciicast/file/#{id}/#{file}" + def json_store_path(%Asciicast{file: v} = asciicast) when is_binary(v) do + file_store_path(asciicast, :file) end - def json_store_path(%__MODULE__{id: id, stdout_frames: stdout_frames}) when is_binary(stdout_frames) do - "asciicast/stdout_frames/#{id}/#{stdout_frames}" + def json_store_path(%Asciicast{stdout_frames: v} = asciicast) when is_binary(v) do + file_store_path(asciicast, :stdout_frames) + end + + def file_store_path(%Asciicast{id: id, file: fname}, :file) do + file_store_path(:file, id, fname) + end + def file_store_path(%Asciicast{id: id, stdout_frames: fname}, :stdout_frames) do + file_store_path(:stdout_frames, id, fname) + end + def file_store_path(%Asciicast{id: id, stdout_data: fname}, :stdout_data) do + file_store_path(:stdout_data, id, fname) + end + def file_store_path(%Asciicast{id: id, stdout_timing: fname}, :stdout_timing) do + file_store_path(:stdout_timing, id, fname) + end + def file_store_path(type, id, fname) when is_binary(fname) do + "asciicast/#{type}/#{id}/#{fname}" end def snapshot_at(%Asciicast{snapshot_at: snapshot_at, duration: duration}) do From 9d7a7d455b5e8efe5a4c44534b4ae6c0f7880e01 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Fri, 16 Jun 2017 09:17:36 +0200 Subject: [PATCH 04/10] Support client v0.9.8 in new upload endpoint --- lib/asciinema/asciicasts.ex | 3 +- test/asciinema/asciicasts_test.exs | 33 ++++++++++++++++++- .../api/asciicast_controller_test.exs | 14 ++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/lib/asciinema/asciicasts.ex b/lib/asciinema/asciicasts.ex index 4014f89..8fdce4b 100644 --- a/lib/asciinema/asciicasts.ex +++ b/lib/asciinema/asciicasts.ex @@ -50,9 +50,10 @@ defmodule Asciinema.Asciicasts do def create_asciicast(user, %{"meta" => attrs, "stdout" => %Plug.Upload{filename: d_filename} = data, - "stdout_timing" => %Plug.Upload{filename: t_filename} = timing}, _user_agent) do + "stdout_timing" => %Plug.Upload{filename: t_filename} = timing}, user_agent) do attrs = Map.put(attrs, "version", 0) asciicast = %Asciicast{user_id: user.id, + user_agent: unless(attrs["uname"], do: user_agent), stdout_data: d_filename, stdout_timing: t_filename, private: user.asciicasts_private_by_default} diff --git a/test/asciinema/asciicasts_test.exs b/test/asciinema/asciicasts_test.exs index f964f56..f2fbca6 100644 --- a/test/asciinema/asciicasts_test.exs +++ b/test/asciinema/asciicasts_test.exs @@ -3,7 +3,7 @@ defmodule Asciinema.AsciicastsTest do alias Asciinema.Asciicasts describe "create_asciicast/2" do - test "json file, v0 format, <= v0.9.7 client" do + test "json file, v0 format with uname" do user = fixture(:user) params = %{"meta" => %{"command" => "/bin/bash", "duration" => 11.146430015564, @@ -35,6 +35,37 @@ defmodule Asciinema.AsciicastsTest do assert asciicast.user_agent == nil end + test "json file, v0 format without uname" do + user = fixture(:user) + params = %{"meta" => %{"command" => "/bin/bash", + "duration" => 11.146430015564, + "shell" => "/bin/zsh", + "terminal_columns" => 96, + "terminal_lines" => 26, + "terminal_type" => "screen-256color", + "title" => "bashing :)"}, + "stdout" => fixture(:upload, %{path: "0.9.8/stdout", + content_type: "application/octet-stream"}), + "stdout_timing" => fixture(:upload, %{path: "0.9.8/stdout.time", + content_type: "application/octet-stream"})} + + {:ok, asciicast} = Asciicasts.create_asciicast(user, params, "a/user/agent") + + assert asciicast.version == 0 + assert asciicast.file == nil + assert asciicast.stdout_data == "stdout" + assert asciicast.stdout_timing == "stdout.time" + assert asciicast.command == "/bin/bash" + assert asciicast.duration == 11.146430015564 + assert asciicast.shell == "/bin/zsh" + assert asciicast.terminal_type == "screen-256color" + assert asciicast.terminal_columns == 96 + assert asciicast.terminal_lines == 26 + assert asciicast.title == "bashing :)" + assert asciicast.uname == nil + assert asciicast.user_agent == "a/user/agent" + end + test "json file, v1 format" do user = fixture(:user) upload = fixture(:upload, %{path: "1/asciicast.json"}) diff --git a/test/controllers/api/asciicast_controller_test.exs b/test/controllers/api/asciicast_controller_test.exs index 742d1f3..a10f0ea 100644 --- a/test/controllers/api/asciicast_controller_test.exs +++ b/test/controllers/api/asciicast_controller_test.exs @@ -31,6 +31,20 @@ defmodule Asciinema.Api.AsciicastControllerTest do assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url end + @tag token: nil + test "separate files (pre-v1 params), v0.9.8 client", %{conn: conn} do + asciicast = %{"meta" => fixture(:upload, %{path: "0.9.8/meta.json", + content_type: "application/json"}), + "stdout" => fixture(:upload, %{path: "0.9.8/stdout", + content_type: "application/octet-stream"}), + "stdout_timing" => fixture(:upload, %{path: "0.9.8/stdout.time", + content_type: "application/octet-stream"})} + + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => asciicast} + assert text_response(conn, 201) =~ @asciicast_url + assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url + end + test "json file, v1 format", %{conn: conn} do upload = fixture(:upload, %{path: "1/asciicast.json"}) conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} From 9f2210ba6899dbaf48fdb730f3dd27064d4728f2 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Fri, 16 Jun 2017 09:26:06 +0200 Subject: [PATCH 05/10] Add test for upload from v0.9.9 client --- test/controllers/api/asciicast_controller_test.exs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/controllers/api/asciicast_controller_test.exs b/test/controllers/api/asciicast_controller_test.exs index a10f0ea..0903767 100644 --- a/test/controllers/api/asciicast_controller_test.exs +++ b/test/controllers/api/asciicast_controller_test.exs @@ -45,6 +45,19 @@ defmodule Asciinema.Api.AsciicastControllerTest do assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url end + test "separate files (pre-v1 params), v0.9.9 client", %{conn: conn} do + asciicast = %{"meta" => fixture(:upload, %{path: "0.9.9/meta.json", + content_type: "application/json"}), + "stdout" => fixture(:upload, %{path: "0.9.9/stdout", + content_type: "application/octet-stream"}), + "stdout_timing" => fixture(:upload, %{path: "0.9.9/stdout.time", + content_type: "application/octet-stream"})} + + conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => asciicast} + assert text_response(conn, 201) =~ @asciicast_url + assert List.first(get_resp_header(conn, "location")) =~ @asciicast_url + end + test "json file, v1 format", %{conn: conn} do upload = fixture(:upload, %{path: "1/asciicast.json"}) conn = post conn, api_asciicast_path(conn, :create), %{"asciicast" => upload} From de9cd8eb60d38b66077129ae4a6f2906453be805 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Fri, 16 Jun 2017 09:38:08 +0200 Subject: [PATCH 06/10] Simplify assertions --- test/asciinema/asciicasts_test.exs | 80 +++++++++++++++--------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/test/asciinema/asciicasts_test.exs b/test/asciinema/asciicasts_test.exs index f2fbca6..bcb5427 100644 --- a/test/asciinema/asciicasts_test.exs +++ b/test/asciinema/asciicasts_test.exs @@ -1,6 +1,6 @@ defmodule Asciinema.AsciicastsTest do use Asciinema.DataCase - alias Asciinema.Asciicasts + alias Asciinema.{Asciicasts, Asciicast} describe "create_asciicast/2" do test "json file, v0 format with uname" do @@ -20,19 +20,19 @@ defmodule Asciinema.AsciicastsTest do {:ok, asciicast} = Asciicasts.create_asciicast(user, params, "a/user/agent") - assert asciicast.version == 0 - assert asciicast.file == nil - assert asciicast.stdout_data == "stdout" - assert asciicast.stdout_timing == "stdout.time" - assert asciicast.command == "/bin/bash" - assert asciicast.duration == 11.146430015564 - assert asciicast.shell == "/bin/zsh" - assert asciicast.terminal_type == "screen-256color" - assert asciicast.terminal_columns == 96 - assert asciicast.terminal_lines == 26 - assert asciicast.title == "bashing :)" - assert asciicast.uname == "Linux 3.9.9-302.fc19.x86_64 #1 SMP Sat Jul 6 13:41:07 UTC 2013 x86_64" - assert asciicast.user_agent == nil + assert %Asciicast{version: 0, + file: nil, + stdout_data: "stdout", + stdout_timing: "stdout.time", + command: "/bin/bash", + duration: 11.146430015564, + shell: "/bin/zsh", + terminal_type: "screen-256color", + terminal_columns: 96, + terminal_lines: 26, + title: "bashing :)", + uname: "Linux 3.9.9-302.fc19.x86_64 #1 SMP Sat Jul 6 13:41:07 UTC 2013 x86_64", + user_agent: nil} = asciicast end test "json file, v0 format without uname" do @@ -51,19 +51,19 @@ defmodule Asciinema.AsciicastsTest do {:ok, asciicast} = Asciicasts.create_asciicast(user, params, "a/user/agent") - assert asciicast.version == 0 - assert asciicast.file == nil - assert asciicast.stdout_data == "stdout" - assert asciicast.stdout_timing == "stdout.time" - assert asciicast.command == "/bin/bash" - assert asciicast.duration == 11.146430015564 - assert asciicast.shell == "/bin/zsh" - assert asciicast.terminal_type == "screen-256color" - assert asciicast.terminal_columns == 96 - assert asciicast.terminal_lines == 26 - assert asciicast.title == "bashing :)" - assert asciicast.uname == nil - assert asciicast.user_agent == "a/user/agent" + assert %Asciicast{version: 0, + file: nil, + stdout_data: "stdout", + stdout_timing: "stdout.time", + command: "/bin/bash", + duration: 11.146430015564, + shell: "/bin/zsh", + terminal_type: "screen-256color", + terminal_columns: 96, + terminal_lines: 26, + title: "bashing :)", + uname: nil, + user_agent: "a/user/agent"} = asciicast end test "json file, v1 format" do @@ -72,19 +72,19 @@ defmodule Asciinema.AsciicastsTest do {:ok, asciicast} = Asciicasts.create_asciicast(user, upload, "a/user/agent") - assert asciicast.version == 1 - assert asciicast.file == "asciicast.json" - assert asciicast.stdout_data == nil - assert asciicast.stdout_timing == nil - assert asciicast.command == "/bin/bash" - assert asciicast.duration == 11.146430015564 - assert asciicast.shell == "/bin/zsh" - assert asciicast.terminal_type == "screen-256color" - assert asciicast.terminal_columns == 96 - assert asciicast.terminal_lines == 26 - assert asciicast.title == "bashing :)" - assert asciicast.uname == nil - assert asciicast.user_agent == "a/user/agent" + assert %Asciicast{version: 1, + file: "asciicast.json", + stdout_data: nil, + stdout_timing: nil, + command: "/bin/bash", + duration: 11.146430015564, + shell: "/bin/zsh", + terminal_type: "screen-256color", + terminal_columns: 96, + terminal_lines: 26, + title: "bashing :)", + uname: nil, + user_agent: "a/user/agent"} = asciicast end test "json file, v1 format (missing required data)" do From b94af03d581789f7fcb9b12e2af5d3ad88af23df Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Fri, 16 Jun 2017 09:40:55 +0200 Subject: [PATCH 07/10] Fix test description --- test/asciinema/asciicasts_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/asciinema/asciicasts_test.exs b/test/asciinema/asciicasts_test.exs index bcb5427..6ec5685 100644 --- a/test/asciinema/asciicasts_test.exs +++ b/test/asciinema/asciicasts_test.exs @@ -2,7 +2,7 @@ defmodule Asciinema.AsciicastsTest do use Asciinema.DataCase alias Asciinema.{Asciicasts, Asciicast} - describe "create_asciicast/2" do + describe "create_asciicast/3" do test "json file, v0 format with uname" do user = fixture(:user) params = %{"meta" => %{"command" => "/bin/bash", From 033744cf1ff5e7fcbfb862ad1877212b5fd2bb01 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sat, 17 Jun 2017 20:23:35 +0200 Subject: [PATCH 08/10] Enqueue asciicast processing job from Elixir --- config/config.exs | 4 ++++ config/test.exs | 2 ++ lib/asciinema.ex | 3 +++ lib/asciinema/asciicasts.ex | 5 +++-- lib/asciinema/asciicasts/poster_generator.ex | 10 ++++++++++ lib/asciinema/asciicasts/poster_generator/noop.ex | 5 +++++ .../asciicasts/poster_generator/sidekiq.ex | 8 ++++++++ lib/asciinema/sidekiq_client.ex | 14 ++++++++++++++ mix.exs | 2 ++ mix.lock | 1 + 10 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 lib/asciinema/asciicasts/poster_generator.ex create mode 100644 lib/asciinema/asciicasts/poster_generator/noop.ex create mode 100644 lib/asciinema/asciicasts/poster_generator/sidekiq.ex create mode 100644 lib/asciinema/sidekiq_client.ex diff --git a/config/config.exs b/config/config.exs index 2b851c9..4015833 100644 --- a/config/config.exs +++ b/config/config.exs @@ -49,6 +49,10 @@ config :asciinema, Asciinema.PngGenerator.A2png, bin_path: System.get_env("A2PNG_BIN_PATH") || "./a2png/a2png.sh", pool_size: String.to_integer(System.get_env("A2PNG_POOL_SIZE") || "2") +config :asciinema, :redis_url, System.get_env("REDIS_URL") || "redis://redis:6379" + +config :asciinema, :poster_generator, Asciinema.Asciicasts.PosterGenerator.Sidekiq + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env}.exs" diff --git a/config/test.exs b/config/test.exs index 09ce1ab..c0717b8 100644 --- a/config/test.exs +++ b/config/test.exs @@ -21,3 +21,5 @@ config :asciinema, Asciinema.Repo, config :asciinema, :file_store, Asciinema.FileStore.Local config :asciinema, Asciinema.FileStore.Local, path: "uploads/test/" + +config :asciinema, :poster_generator, Asciinema.Asciicasts.PosterGenerator.Noop diff --git a/lib/asciinema.ex b/lib/asciinema.ex index dfbc22c..94831e0 100644 --- a/lib/asciinema.ex +++ b/lib/asciinema.ex @@ -6,6 +6,8 @@ defmodule Asciinema do def start(_type, _args) do import Supervisor.Spec + redis_url = Application.get_env(:asciinema, :redis_url) + # Define workers and child supervisors to be supervised children = [ # Start the Ecto repository @@ -15,6 +17,7 @@ defmodule Asciinema do # Start your own worker by calling: Asciinema.Worker.start_link(arg1, arg2, arg3) # worker(Asciinema.Worker, [arg1, arg2, arg3]), :poolboy.child_spec(:worker, Asciinema.PngGenerator.A2png.poolboy_config(), []), + worker(Redix, [redis_url, [name: :redix]]) ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/lib/asciinema/asciicasts.ex b/lib/asciinema/asciicasts.ex index 8fdce4b..e385744 100644 --- a/lib/asciinema/asciicasts.ex +++ b/lib/asciinema/asciicasts.ex @@ -1,6 +1,7 @@ defmodule Asciinema.Asciicasts do import Ecto.Query, warn: false alias Asciinema.{Repo, Asciicast, FileStore} + alias Asciinema.Asciicasts.PosterGenerator def get_asciicast!(id) when is_integer(id) do Repo.get!(Asciicast, id) @@ -93,7 +94,7 @@ defmodule Asciinema.Asciicasts do :ok = FileStore.put_file(file_store_path, tmp_file_path, content_type) end - defp generate_poster(_asciicast) do - # TODO + defp generate_poster(asciicast) do + PosterGenerator.generate(asciicast) end end diff --git a/lib/asciinema/asciicasts/poster_generator.ex b/lib/asciinema/asciicasts/poster_generator.ex new file mode 100644 index 0000000..0ed2dad --- /dev/null +++ b/lib/asciinema/asciicasts/poster_generator.ex @@ -0,0 +1,10 @@ +defmodule Asciinema.Asciicasts.PosterGenerator do + alias Asciinema.Asciicast + + @doc "Generates poster for given asciicast" + @callback generate(asciicast :: %Asciicast{}) :: :ok | {:error, term} + + def generate(asciicast) do + Application.get_env(:asciinema, :poster_generator).generate(asciicast) + end +end diff --git a/lib/asciinema/asciicasts/poster_generator/noop.ex b/lib/asciinema/asciicasts/poster_generator/noop.ex new file mode 100644 index 0000000..338daa4 --- /dev/null +++ b/lib/asciinema/asciicasts/poster_generator/noop.ex @@ -0,0 +1,5 @@ +defmodule Asciinema.Asciicasts.PosterGenerator.Noop do + def generate(_asciicast) do + :ok + end +end diff --git a/lib/asciinema/asciicasts/poster_generator/sidekiq.ex b/lib/asciinema/asciicasts/poster_generator/sidekiq.ex new file mode 100644 index 0000000..c69f457 --- /dev/null +++ b/lib/asciinema/asciicasts/poster_generator/sidekiq.ex @@ -0,0 +1,8 @@ +defmodule Asciinema.Asciicasts.PosterGenerator.Sidekiq do + alias Asciinema.Asciicast + alias Asciinema.SidekiqClient + + def generate(%Asciicast{id: id}) do + SidekiqClient.enqueue("AsciicastWorker", [id]) + end +end diff --git a/lib/asciinema/sidekiq_client.ex b/lib/asciinema/sidekiq_client.ex new file mode 100644 index 0000000..af6d665 --- /dev/null +++ b/lib/asciinema/sidekiq_client.ex @@ -0,0 +1,14 @@ +defmodule Asciinema.SidekiqClient do + def enqueue(class, args, queue \\ "default") do + job = %{queue: queue, + class: class, + args: args, + enqueued_at: Timex.now |> Timex.to_unix, + jid: Crypto.random_token(24), + retry: true} + + payload = Poison.encode!(job) + {:ok, _} = Redix.command(:redix, ["LPUSH", "queue:#{queue}", payload]) + :ok + end +end diff --git a/mix.exs b/mix.exs index 372049c..aa7406a 100644 --- a/mix.exs +++ b/mix.exs @@ -33,6 +33,7 @@ defmodule Asciinema.Mixfile do :poolboy, :porcelain, :postgrex, + :redix, :timex, :timex_ecto, ]] @@ -63,6 +64,7 @@ defmodule Asciinema.Mixfile do {:poolboy, "~> 1.5"}, {:porcelain, "~> 2.0"}, {:postgrex, ">= 0.0.0"}, + {:redix, ">= 0.6.1"}, {:timex, "~> 3.0"}, {:timex_ecto, "~> 3.0"}, ] diff --git a/mix.lock b/mix.lock index 2daa307..4432d88 100644 --- a/mix.lock +++ b/mix.lock @@ -32,6 +32,7 @@ "porcelain": {:hex, :porcelain, "2.0.3", "2d77b17d1f21fed875b8c5ecba72a01533db2013bd2e5e62c6d286c029150fdc", [:mix], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.11.2", "139755c1359d3c5c6d6e8b1ea72556d39e2746f61c6ddfb442813c91f53487e8", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 1.0-rc", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm"}, + "redix": {:hex, :redix, "0.6.1", "20986b0e02f02b13e6f53c79a1ae70aa83147488c408f40275ec261f5bb0a6d0", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}, "timex": {:hex, :timex, "3.1.15", "94abaec8fef2436ced4d0e1b4ed50c8eaa5fb9138fc0699946ddee7abf5aaff2", [:mix], [{:combine, "~> 0.7", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "timex_ecto": {:hex, :timex_ecto, "3.0.5", "3ec6c25e10d2c0020958e5df64d2b5e690e441faa2c2259da8bc6bd3d7f39256", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.0", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, From 99c326780fd7c9643cf13980f7f2348d7dbacf35 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sat, 17 Jun 2017 20:31:21 +0200 Subject: [PATCH 09/10] Route to new upload endpoint --- docker/nginx/asciinema.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/asciinema.conf b/docker/nginx/asciinema.conf index 449d657..1a8b655 100644 --- a/docker/nginx/asciinema.conf +++ b/docker/nginx/asciinema.conf @@ -24,7 +24,7 @@ server { client_max_body_size 16m; - location ~ ^/(phoenix/|css/|js/|images/|fonts/|docs/?) { + location ~ ^/(phoenix/|css/|js/|images/|fonts/|docs/?|api/asciicasts) { try_files /maintenance.html $uri/index.html $uri.html $uri @phoenix; } From f07bedc754e7f5cea28d6fb146208096e36f7518 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 18 Jun 2017 12:27:49 +0200 Subject: [PATCH 10/10] Workaround 1.3.0-1.4.0 client bug regarding basic auth header --- lib/asciinema/auth.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/asciinema/auth.ex b/lib/asciinema/auth.ex index 8d74605..54145b3 100644 --- a/lib/asciinema/auth.ex +++ b/lib/asciinema/auth.ex @@ -14,6 +14,7 @@ defmodule Asciinema.Auth do def get_basic_auth(conn) do with ["Basic " <> auth] <- Conn.get_req_header(conn, "authorization"), + auth = String.replace(auth, ~r/^%/, ""), # workaround for 1.3.0-1.4.0 client bug {:ok, username_password} <- Base.decode64(auth), [username, password] <- String.split(username_password, ":") do {username, password}