From 64bd4313cb4139336bd6afb18ccf1642e10e34c1 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 13 Dec 2024 09:55:45 +0000 Subject: [PATCH 01/58] outline of functions required for #17 --- .env_sample | 2 +- lib/app/user.ex | 32 +++++++++++++++++++++++++++++ lib/app_web/live/app_live.html.heex | 2 +- mix.exs | 2 +- mix.lock | 25 ++++++++++------------ test/app/user_test.exs | 13 +++++++++++- test/app_web/live/app_live_test.exs | 4 ++-- 7 files changed, 60 insertions(+), 20 deletions(-) diff --git a/.env_sample b/.env_sample index a66e7d7..edb48a2 100644 --- a/.env_sample +++ b/.env_sample @@ -1,6 +1,6 @@ export ENCRYPTION_KEYS='nMdayQpR0aoasLaq1g94FLba+A+wB44JLko47sVQXMg=,L+ZVX8iheoqgqb22mUpATmMDsvVGtafoAeb0KN5uWf0=' export SECRET_KEY_BASE=2PzB7PPnpuLsbWmWtXpGyI+kfSQSQ1zUW2Atz/+8PdZuSEJzHgzGnJWV35nTKRwx -export AUTH_API_KEY=88SwQGzxQEvo6S9Pu7FZGp9btNo52rVkwtrhyub9i6K6UxVqho9A/88SwQGswuPR1uYWEjFm8tBeHjXQ7LMnc5p6deCs3H2Fb8vbUWf8t/dwylauth.herokuapp.com +export AUTH_API_KEY=YTsV7fG5mZ2KRWmvE3u431sZYsaZhhC8oqvQSDg85VnqMQXSDEBjh/YTsV7TBnHp1yxy2LLxBZYyVBrTYPtiKjLbApKiFkva3YQ8rrGgYeV/authdemo.fly.dev # https://github.com/settings/tokens/new export GH_PERSONAL_ACCESS_TOKEN=YourTokenHere \ No newline at end of file diff --git a/lib/app/user.ex b/lib/app/user.ex index 8e116ca..c5b573e 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -39,4 +39,36 @@ defmodule App.User do |> changeset(attrs) |> Repo.insert() end + + # Envar.require_env_file(".env") + + def get_org_members_from_api(org_name) do + # dbg(org_name) + token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") + + client = Tentacat.Client.new(%{access_token: token}) + {200, data, _res} = Tentacat.Organizations.Members.list(client, org_name) + dbg(data) + Useful.atomize_map_keys(data) + end + + def get_user_from_api(username) do + token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") + client = Tentacat.Client.new(%{access_token: token}) + {200, data, _res} = Tentacat.Users.find(client, username) + Useful.atomize_map_keys(data) + end + + # Next: get list of org members + # get user for each in the list + # map data to our table + # insert data + + def map_github_user_fieldsre(entry) do + %{ + + } + end + + end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index d84624f..eb5be82 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -1,4 +1,4 @@

- LiveView App Page with Tailwind! + hello david

\ No newline at end of file diff --git a/mix.exs b/mix.exs index 9cc5111..0aec3f1 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,7 @@ defmodule App.MixProject do # Type `mix help deps` for examples and options. defp deps do [ - {:phoenix, "~> 1.7.1"}, + {:phoenix, "~> 1.7.17"}, {:phoenix_ecto, "~> 4.4"}, {:ecto_sql, "~> 3.6"}, {:postgrex, ">= 0.0.0"}, diff --git a/mix.lock b/mix.lock index 59471e2..c9c97b7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,12 +1,10 @@ %{ "argon2_elixir": {:hex, :argon2_elixir, "3.0.0", "fd4405f593e77b525a5c667282172dd32772d7c4fa58cdecdaae79d2713b6c5f", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "8b753b270af557d51ba13fcdebc0f0ab27a2a6792df72fd5a6cf9cfaffcedc57"}, - "atomic_map": {:hex, :atomic_map, "0.9.3", "3c7f1302e0590164732d08ca999708efbb2cd768abf2911cf140280ce2dc499d", [:mix], [], "hexpm", "c237babf301bd2435bd85b96cffc973022b4cbb7721537059ee0dd3bb74938d2"}, "auth_plug": {:hex, :auth_plug, "1.5.0", "fa9f8c022d76cd7ef4cd322f25846698f18a0b6d2435a3ae1cb0e61167199e52", [:mix], [{:envar, "~> 1.0.8", [hex: :envar, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.8.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.5", [hex: :joken, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:useful, "~> 1.0.8", [hex: :useful, repo: "hexpm", optional: false]}], "hexpm", "3481aed63f8bb24081268db6713c931c33b5b3b17d2f53ca7949ff8443c1fbb7"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.11", "4bbd584741601eb658007339ea730b082cc61f3554cf2e8f39bf693a11b49073", [:mix], [], "hexpm", "e03990b4db988df56262852f20de0f659871c35154691427a5047f4967a16a62"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, - "comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"}, - "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, + "comeonin": {:hex, :comeonin, "5.5.0", "364d00df52545c44a139bad919d7eacb55abf39e86565878e17cebb787977368", [:mix], [], "hexpm", "6287fc3ba0aad34883cbe3f7949fc1d1e738e5ccdce77165bc99490aa69f47fb"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, @@ -14,9 +12,9 @@ "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"}, - "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, + "ecto": {:hex, :ecto, "3.12.5", "4a312960ce612e17337e7cefcf9be45b95a3be6b36b6f94dfb3d8c361d631866", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6eb18e80bef8bb57e17f5a7f068a1719fbda384d40fc37acb8eb8aeca493b6ea"}, "ecto_sql": {:hex, :ecto_sql, "3.12.1", "c0d0d60e85d9ff4631f12bafa454bc392ce8b9ec83531a412c12a0d415a3a4d0", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aff5b958a899762c5f09028c847569f7dfb9cc9d63bdb8133bff8a5546de6bf5"}, - "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, + "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, "envar": {:hex, :envar, "1.0.9", "b51976b00035efd254c3f51ee7f3cf2e22f91350ef104da393d1d71286eb4fdc", [:mix], [], "hexpm", "bfc3a73f97910c744e0d9e53722ad2d1f73bbb392d2dd1cac63e0af27776fde3"}, "esbuild": {:hex, :esbuild, "0.9.0", "f043eeaca4932ca8e16e5429aebd90f7766f31ac160a25cbd9befe84f2bc068f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b415027f71d5ab57ef2be844b2a10d0c1b5a492d431727f43937adce22ba45ae"}, "ex_doc": {:hex, :ex_doc, "0.37.1", "65ca30d242082b95aa852b3b73c9d9914279fff56db5dc7b3859be5504417980", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6774f75477733ea88ce861476db031f9399c110640752ca2b400dbbb50491224"}, @@ -24,22 +22,21 @@ "fields": {:hex, :fields, "2.10.3", "2683d8fdfd582869b459c88c693c4e15e2247961869719e03213286764b82093", [:mix], [{:argon2_elixir, "~> 3.0.0", [hex: :argon2_elixir, repo: "hexpm", optional: false]}, {:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:envar, "~> 1.0.8", [hex: :envar, repo: "hexpm", optional: false]}, {:html_sanitize_ex, "~> 1.4.2", [hex: :html_sanitize_ex, repo: "hexpm", optional: false]}], "hexpm", "d68567e175fb6d3be04cdf795fc6ed94973c66706ca3f87e2ec6251159b4b965"}, "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, "floki": {:hex, :floki, "0.37.0", "b83e0280bbc6372f2a403b2848013650b16640cd2470aea6701f0632223d719e", [:mix], [], "hexpm", "516a0c15a69f78c47dc8e0b9b3724b29608aa6619379f91b1ffa47109b5d0dd3"}, - "hackney": {:hex, :hackney, "1.18.2", "d7ff544ddae5e1cb49e9cf7fa4e356d7f41b283989a1c304bfc47a8cc1cf966f", [:rebar3], [{:certifi, "~>2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "af94d5c9f97857db257090a4a10e5426ecb6f4918aa5cc666798566ae14b65fd"}, - "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, - "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.2", "c479398b6de798c03eb5d04a0a9a9159d73508f83f6590a00b8eacba3619cf4c", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "aef6c28585d06a9109ad591507e508854c5559561f950bbaea773900dd369b0e"}, + "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, + "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.3", "67b3d9fa8691b727317e0cc96b9b3093be00ee45419ffb221cdeee88e75d1360", [:mix], [{:mochiweb, "~> 2.15 or ~> 3.1", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "87748d3c4afe949c7c6eb7150c958c2bcba43fc5b2a02686af30e636b74bccb7"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, + "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"}, - "jose": {:hex, :jose, "1.11.2", "f4c018ccf4fdce22c71e44d471f15f723cb3efab5d909ab2ba202b5bf35557b3", [:mix, :rebar3], [], "hexpm", "98143fbc48d55f3a18daba82d34fe48959d44538e9697c08f34200fa5f0947d2"}, + "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, + "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mochiweb": {:hex, :mochiweb, "2.22.0", "f104d6747c01a330c38613561977e565b788b9170055c5241ac9dd6e4617cba5", [:rebar3], [], "hexpm", "cbbd1fd315d283c576d1c8a13e0738f6dafb63dc840611249608697502a07655"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, + "mochiweb": {:hex, :mochiweb, "3.2.2", "bb435384b3b9fd1f92f2f3fe652ea644432877a3e8a81ed6459ce951e0482ad3", [:rebar3], [], "hexpm", "4114e51f1b44c270b3242d91294fe174ce1ed989100e8b65a1fab58e0cba41d5"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "phoenix": {:hex, :phoenix, "1.7.19", "36617efe5afbd821099a8b994ff4618a340a5bfb25531a1802c4d4c634017a57", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "ba4dc14458278773f905f8ae6c2ec743d52c3a35b6b353733f64f02dfe096cd6"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, diff --git a/test/app/user_test.exs b/test/app/user_test.exs index 7396176..d9e485d 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -1,5 +1,6 @@ defmodule App.UserTest do use App.DataCase + # alias App.User test "App.User.create/1" do user = %{ @@ -24,4 +25,14 @@ defmodule App.UserTest do assert {:ok, inserted_user} = App.User.create(user) assert inserted_user.name == user.name end -end \ No newline at end of file + + test "get_org_members_from_api/1" do + App.User.get_org_members_from_api("dwyl") |> dbg + assert true == true + end + + test "get_user_from_api/1" do + data = App.User.get_user_from_api("iteles") |> dbg + assert data.public_repos > 30 + end +end diff --git a/test/app_web/live/app_live_test.exs b/test/app_web/live/app_live_test.exs index 6ceade5..37ef70f 100644 --- a/test/app_web/live/app_live_test.exs +++ b/test/app_web/live/app_live_test.exs @@ -3,6 +3,6 @@ defmodule AppWeb.AppLiveTest do test "GET /", %{conn: conn} do conn = get(conn, "/") - assert html_response(conn, 200) =~ "LiveView App Page" + assert html_response(conn, 200) =~ "hello david" end -end \ No newline at end of file +end From be8858977acaa1a4afbb01d82573fcf8ccf2263d Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 13 Dec 2024 19:04:05 +0000 Subject: [PATCH 02/58] explicitly export GH_PERSONAL_ACCESS_TOKEN in ci.yml for #208 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2860ce..91d7269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: MIX_ENV: test AUTH_API_KEY: ${{ secrets.AUTH_API_KEY }} ENCRYPTION_KEYS: ${{ secrets.ENCRYPTION_KEYS }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 37e070d261692e9069242c47ccd2bd2f4d7c8e04 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sat, 14 Dec 2024 09:20:11 +0000 Subject: [PATCH 03/58] insert github user #17 --- lib/app/user.ex | 28 ++++++++++++++++++++++++---- test/app/user_test.exs | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/lib/app/user.ex b/lib/app/user.ex index c5b573e..a7c7d6d 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -48,7 +48,7 @@ defmodule App.User do client = Tentacat.Client.new(%{access_token: token}) {200, data, _res} = Tentacat.Organizations.Members.list(client, org_name) - dbg(data) + # dbg(data) Useful.atomize_map_keys(data) end @@ -56,7 +56,12 @@ defmodule App.User do token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") client = Tentacat.Client.new(%{access_token: token}) {200, data, _res} = Tentacat.Users.find(client, username) - Useful.atomize_map_keys(data) + {:ok, entry} = Useful.atomize_map_keys(data) + |> dbg + |> map_github_user_fields_to_table() + |> create() + + entry end # Next: get list of org members @@ -64,9 +69,24 @@ defmodule App.User do # map data to our table # insert data - def map_github_user_fieldsre(entry) do + def map_github_user_fields_to_table(u) do %{ - + id: u.id, + avatar_url: String.split(u.avatar_url, "?") |> List.first, + bio: u.bio, + blog: u.blog, + company: String.replace(u.company, "@", ""), + created_at: u.created_at, + email: u.email, + followers: u.followers, + following: u.following, + hireable: u.hireable, + location: u.location, + login: u.login, + name: u.name, + public_repos: u.public_repos, + two_factor_authentication: false, + updated_at: u.updated_at } end diff --git a/test/app/user_test.exs b/test/app/user_test.exs index d9e485d..1d09539 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -27,7 +27,7 @@ defmodule App.UserTest do end test "get_org_members_from_api/1" do - App.User.get_org_members_from_api("dwyl") |> dbg + App.User.get_org_members_from_api("dwyl") # |> dbg assert true == true end @@ -36,3 +36,39 @@ defmodule App.UserTest do assert data.public_repos > 30 end end + +%{ + organizations_url: "https://api.github.com/users/iteles/orgs", + following: 80, + login: "iteles", + public_repos: 31, + received_events_url: "https://api.github.com/users/iteles/received_events", + bio: "Co-founder @dwyl \r\n", + user_view_type: "public", + company: "@dwyl", + gravatar_id: "", + twitter_username: nil, + following_url: "https://api.github.com/users/iteles/following{/other_user}", + created_at: "2013-04-17T21:10:06Z", + followers: 400, + site_admin: false, + blog: "http://www.twitter.com/iteles", + starred_url: "https://api.github.com/users/iteles/starred{/owner}{/repo}", + public_gists: 0, + hireable: true, + gists_url: "https://api.github.com/users/iteles/gists{/gist_id}", + events_url: "https://api.github.com/users/iteles/events{/privacy}", + followers_url: "https://api.github.com/users/iteles/followers", + node_id: "MDQ6VXNlcjQxODUzMjg=", + url: "https://api.github.com/users/iteles", + id: 4185328, + name: "Ines Teles Correia", + subscriptions_url: "https://api.github.com/users/iteles/subscriptions", + html_url: "https://github.com/iteles", + location: "London, UK", + type: "User", + avatar_url: "https://avatars.githubusercontent.com/u/4185328?v=4", + email: nil, + repos_url: "https://api.github.com/users/iteles/repos", + updated_at: "2024-08-05T22:59:09Z" +} From 4e108adb1d1151fbf5bf99b3cc5c3cd3377882b5 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 15 Dec 2024 08:39:44 +0000 Subject: [PATCH 04/58] Repo.insert(on_conflict: :replace_all, conflict_target: [:id]) --- lib/app/user.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/app/user.ex b/lib/app/user.ex index a7c7d6d..57df4ee 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -37,13 +37,12 @@ defmodule App.User do def create(attrs) do %User{} |> changeset(attrs) - |> Repo.insert() + |> Repo.insert(on_conflict: :replace_all, conflict_target: [:id]) end # Envar.require_env_file(".env") def get_org_members_from_api(org_name) do - # dbg(org_name) token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") client = Tentacat.Client.new(%{access_token: token}) From b623f0113e613209aaa382661c5dfa3f876b842d Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 18 Dec 2024 18:26:00 +0000 Subject: [PATCH 05/58] tidy up #1 #211 --- BUILDIT.md | 611 +++++----------------------------------------- lib/app/github.ex | 3 +- 2 files changed, 58 insertions(+), 556 deletions(-) diff --git a/BUILDIT.md b/BUILDIT.md index 4a8aa5a..aad1715 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -1,43 +1,47 @@
-# Build Log πŸ‘©β€πŸ’» +# Build Log πŸ‘©πŸ»β€πŸ’» [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/dwyl/who/ci.yml?label=build&style=flat-square&branch=main)](https://github.com/dwyl/who/actions/workflows/ci.yml) [![codecov.io](https://img.shields.io/codecov/c/github/dwyl/who/main.svg?style=flat-square)](http://codecov.io/github/dwyl/who?branch=main) -[![Hex.pm](https://img.shields.io/hexpm/v/elixir_auth_google?color=brightgreen&style=flat-square)](https://hex.pm/packages/elixir_auth_google) +[![Hex.pm](https://img.shields.io/hexpm/v/phoenix?color=brightgreen&style=flat-square)](https://hex.pm/packages/phoenix) [![contributions welcome](https://img.shields.io/badge/feedback-welcome-brightgreen.svg?style=flat-square)](https://github.com/dwyl/who/issues) [![HitCount](https://hits.dwyl.com/dwyl/who-buildit.svg)](https://hits.dwyl.com/dwyl/who-buildit) -This is a log -of the steps taken +This is a log +of the steps taken to build the **`WHO`** App. πŸš€
-It took us _hours_ +It took us _hours_ to write it, -but you can -[***speedrun***](https://en.wikipedia.org/wiki/Speedrun) +but you can +[_**speedrun**_](https://en.wikipedia.org/wiki/Speedrun) it in **10 minutes**. 🏁
-> **Note**: we have referenced sections +> **Note**: we have referenced sections > in our more extensive tutorials/examples -> to keep this doc +> to keep this doc > [DRY](https://en.wikipedia.org/wiki/Don't_repeat_yourself).
> You don't have to follow every step in > the other tutorials/examples, > but they are linked in case you get stuck. -In this log we have written the -"[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete)" +In this log we have written the +"[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete)" functions first and _then_ built the interface.
We were able to do this because we had a good idea of which functions we were going to need.
If you are reading through this -and scratching your head +and scratching your head wondering where a particular function will be used, simply scroll down to the interface section -where (_hopefully_) it will all be clear. +where (_hopefully_) it will all be clear. + +> **Note**: if anything is _still_ unclear, +> please open an issue: +> [dwyl/who/issues](https://github.com/dwyl/who/issues) At the end of each step, remember to run the tests: @@ -53,11 +57,10 @@ We suggest keeping two terminal tabs/windows running;
one for the server `mix phx.server` and the other for the **tests**.
That way you can also see the UI as you progress. -With that in place, let's get building! - - +With that in place, let's get building! -- [Build Log πŸ‘©β€πŸ’»](#build-log-) +- [Build Log πŸ‘©πŸ»β€πŸ’»](#build-log-) +- [0. Prerequisites: _Before_ You Start](#0-prerequisites-before-you-start) - [1. Create a New `Phoenix` App](#1-create-a-new-phoenix-app) - [1.1 Run the `Phoenix` App](#11-run-the-phoenix-app) - [1.2 Run the tests:](#12-run-the-tests) @@ -76,18 +79,11 @@ With that in place, let's get building! - [2.2.3 Re-run the Tests](#223-re-run-the-tests) - [3. Setup `GitHub` API](#3-setup-github-api) - [3.1 Make the `user` Tests Pass](#31-make-the-user-tests-pass) -- [4. Create `Timer`](#4-create-timer) - - [Make `timer` tests pass](#make-timer-tests-pass) -- [5. `items` with `timers`](#5-items-with-timers) - - [5.1 Test for `accumulate_item_timers/1`](#51-test-for-accumulate_item_timers1) - - [5.2 Implement the `accummulate_item_timers/1` function](#52-implement-the-accummulate_item_timers1-function) - - [5.3 Test for `items_with_timers/1`](#53-test-for-items_with_timers1) - - [5.4 Implement `items_with_timers/1`](#54-implement-items_with_timers1) -- [6. Add Authentication](#6-add-authentication) - - [6.1 Add `auth_plug` to `deps`](#61-add-auth_plug-to-deps) - - [6.2 Get your `AUTH_API_KEY`](#62-get-your-auth_api_key) - - [6.2 Create Auth Controller](#62-create-auth-controller) -- [7. Create `LiveView` Functions](#7-create-liveview-functions) +- [X. Add Authentication](#x-add-authentication) + - [X.1 Add `auth_plug` to `deps`](#x1-add-auth_plug-to-deps) + - [X.2 Get your `AUTH_API_KEY`](#x2-get-your-auth_api_key) + - [X.3 Create Auth Controller](#x3-create-auth-controller) +- [Y. Create `LiveView` Functions](#y-create-liveview-functions) - [7.1 Write `LiveView` Tests](#71-write-liveview-tests) - [7.2 Implement the `LiveView` functions](#72-implement-the-liveview-functions) - [8. Implement the `LiveView` UI Template](#8-implement-the-liveview-ui-template) @@ -106,10 +102,17 @@ With that in place, let's get building! - [12.2 Run The App](#122-run-the-app) - [Thanks!](#thanks) +# 0. Prerequisites: _Before_ You Start + +_Before_ you dive in, +make sure you have `Phoenix` and `Postgres` installed, +see how at: +[dwyl/phoenix#how](https://github.com/dwyl/phoenix-chat-example?tab=readme-ov-file#how) + # 1. Create a New `Phoenix` App -Open your terminal and +Open your terminal and **create** a **new `Phoenix` app** with the following command: @@ -434,24 +437,27 @@ This app stores data in **five** schemas: 5. `follows` - https://docs.github.com/en/rest/users/followers - List the `people` a `user` follows. For each of these schemas we are storing -a _subset_ of the data; +a _subset_ of the data; only what we need right now.
-We can always add more -("[backfill](https://stackoverflow.com/questions/70871818/what-is-backfilling-in-data)") -later as needed. - +We can always add more +("[backfill](https://stackoverflow.com/questions/70871818/what-is-backfilling-in-data)") +_later_ as needed. -Create database schemas +Create database schemas to store the data -with the following +with the following [**`mix phx.gen.schema`**](https://hexdocs.pm/phoenix/Mix.Tasks.Phx.Gen.Schema.html) commands: ```sh mix phx.gen.schema User users login:string avatar_url:string name:string company:string bio:string blog:string location:string email:string created_at:string two_factor_authentication:boolean followers:integer following:integer + mix phx.gen.schema Org orgs login:string avatar_url:string name:string company:string public_repos:integer location:string description:string followers:integer + mix phx.gen.schema Repository repositories name:string full_name:string owner_id:integer owner_name:string description:string fork:boolean forks_count:integer watchers_count:integer stargazers_count:integer topics:string open_issues_count:integer created_at:string pushed_at:string + mix phx.gen.schema Star stars repo_id:integer user_id:integer stop:utc_datetime + mix phx.gen.schema Follow follows follower_id:integer following_id:integer stop:utc_datetime ``` @@ -505,22 +511,23 @@ Specifically the files: `lib/app/repository.ex` and `lib/app/user.ex` -have **_zero_ test coverage**. +have **_zero_ test coverage**. We will address this test coverage shortfall in the next section. -Yes, we _know_ this is not -["TDD"](https://github.com/dwyl/learn-tdd#what-is-tdd) +Yes, we _know_ this is _not_ +[**TDD**](https://github.com/dwyl/learn-tdd#what-is-tdd) because we aren't writing the tests _first_. But by creating database schemas, -we have a scaffold +we have a scaffold for the next stage. -See: https://en.wikipedia.org/wiki/Scaffold_(programming) +See: +https://en.wikipedia.org/wiki/Scaffold_(programming)
## 2.2 Write Tests for Schema/Scaffold Code -The **`mix phx.gen.schema`** command +The **`mix phx.gen.schema`** command doesn't create the test files for the schemas. We can create these quick. @@ -658,10 +665,9 @@ If you get stuck, you can always refer to the file in the finished project: [`/lib/app/repository.ex`](https://github.com/dwyl/who/blob/56e3445a37fff07f4e7e8561083d7ec77296ed3f/lib/app/repository.ex) - ### 2.2.3 Re-run the Tests -At this point, +At this point, if you re-run _all_ the tests with coverage: ```sh @@ -876,515 +882,12 @@ end Once you have saved the file, re-run the tests. They should now pass. - -# 4. Create `Timer` - -Open the `test/app/timer_test.exs` file and add the following tests: - -```elixir -defmodule App.TimerTest do - use App.DataCase - alias App.{Item, Timer} - - describe "timers" do - @valid_item_attrs %{text: "some text", person_id: 1} - - test "Timer.start/1 returns timer that has been started" do - {:ok, item} = Item.create_item(@valid_item_attrs) - assert Item.get_item!(item.id).text == item.text - - started = NaiveDateTime.utc_now() - - {:ok, timer} = - Timer.start(%{item_id: item.id, person_id: 1, start: started}) - - assert NaiveDateTime.diff(timer.start, started) == 0 - end - - test "Timer.stop/1 stops the timer that had been started" do - {:ok, item} = Item.create_item(@valid_item_attrs) - assert Item.get_item!(item.id).text == item.text - - {:ok, started} = - NaiveDateTime.new(Date.utc_today, Time.add(Time.utc_now, -1)) - - {:ok, timer} = - Timer.start(%{item_id: item.id, person_id: 1, start: started}) - - assert NaiveDateTime.diff(timer.start, started) == 0 - - ended = NaiveDateTime.utc_now() - {:ok, timer} = Timer.stop(%{id: timer.id, stop: ended}) - assert NaiveDateTime.diff(timer.stop, timer.start) == 1 - end - - test "stop_timer_for_item_id(item_id) should stop the active timer (happy path)" do - {:ok, item} = Item.create_item(@valid_item_attrs) - {:ok, seven_seconds_ago} = - NaiveDateTime.new(Date.utc_today, Time.add(Time.utc_now, -7)) - # Start the timer 7 seconds ago: - {:ok, timer} = - Timer.start(%{item_id: item.id, person_id: 1, start: seven_seconds_ago}) - - #Β stop the timer based on it's item_id - Timer.stop_timer_for_item_id(item.id) - - stopped_timer = Timer.get_timer!(timer.id) - assert NaiveDateTime.diff(stopped_timer.start, seven_seconds_ago) == 0 - assert NaiveDateTime.diff(stopped_timer.stop, stopped_timer.start) == 7 - end - - test "stop_timer_for_item_id(item_id) should not explode if there is no timer (unhappy path)" do - zero_item_id = 0 # random int - Timer.stop_timer_for_item_id(zero_item_id) - assert "Don't stop believing!" - end - - test "stop_timer_for_item_id(item_id) should not melt down if item_id is nil (sad path)" do - nil_item_id = nil # random int - Timer.stop_timer_for_item_id(nil_item_id) - assert "Keep on truckin'" - end - end -end -``` - -## Make `timer` tests pass - -Open the `lib/app/timer.ex` file -and replace the contents with the following code: - -```elixir -defmodule App.Timer do - use Ecto.Schema - import Ecto.Changeset - # import Ecto.Query - alias App.Repo - alias __MODULE__ - require Logger - - schema "timers" do - field :item_id, :id - field :start, :naive_datetime - field :stop, :naive_datetime - - timestamps() - end - - @doc false - def changeset(timer, attrs) do - timer - |> cast(attrs, [:item_id, :start, :stop]) - |> validate_required([:item_id, :start]) - end - - @doc """ - `get_timer/1` gets a single Timer. - - Raises `Ecto.NoResultsError` if the Timer does not exist. - - ## Examples - - iex> get_timer!(123) - %Timer{} - """ - def get_timer!(id), do: Repo.get!(Timer, id) - - - @doc """ - `start/1` starts a timer. - - ## Examples - - iex> start(%{item_id: 1, }) - {:ok, %Timer{start: ~N[2022-07-11 04:20:42]}} - - """ - def start(attrs \\ %{}) do - %Timer{} - |> changeset(attrs) - |> Repo.insert() - end - - @doc """ - `stop/1` stops a timer. - - ## Examples - - iex> stop(%{id: 1}) - {:ok, %Timer{stop: ~N[2022-07-11 05:15:31], etc.}} - - """ - def stop(attrs \\ %{}) do - get_timer!(attrs.id) - |> changeset(%{stop: NaiveDateTime.utc_now}) - |> Repo.update() - end - - @doc """ - `stop_timer_for_item_id/1` stops a timer for the given item_id if there is one. - Fails silently if there is no timer for the given item_id. - - ## Examples - - iex> stop_timer_for_item_id(42) - {:ok, %Timer{item_id: 42, stop: ~N[2022-07-11 05:15:31], etc.}} - - """ - def stop_timer_for_item_id(item_id) when is_nil(item_id) do - Logger.debug("stop_timer_for_item_id/1 called without item_id: #{item_id} fail.") - end - - def stop_timer_for_item_id(item_id) do - # get timer by item_id find the latest one that has not been stopped: - sql = """ - SELECT t.id FROM timers t - WHERE t.item_id = $1 - AND t.stop IS NULL - ORDER BY t.id - DESC LIMIT 1; - """ - res = Ecto.Adapters.SQL.query!(Repo, sql, [item_id]) - - if res.num_rows > 0 do - # IO.inspect(res.rows) - timer_id = res.rows |> List.first() |> List.first() - Logger.debug("Found timer.id: #{timer_id} for item: #{item_id}, attempting to stop.") - stop(%{id: timer_id}) - else - Logger.debug("No active timers found for item: #{item_id}") - end - end -end -``` - -The first few functions are simple again. -The more advanced function is `stop_timer_for_item_id/1`. -The _reason_ for the function is, -as it's name suggests, -to stop a `timer` for an `item` by its' `item_id`. - -We have written the function using "raw" `SQL` -so that it's easier for people who are `new` -to `Phoenix`, and _specifically_ `Ecto` to understand. - -# 5. `items` with `timers` - -The _interesting_ thing we are UX-testing in the MVP -is the _combination_ of (todo list) `items` and `timers`. - -So we need a way of:
-**a.** Selecting all the `timers` for a given `item`
-**b.** Accumulating the `timers` for the `item`
- -> **Note**: We would have _loved_ -to find a single `Ecto` function to do this, -but we didn't. -If you know of one, -please share! - - -## 5.1 Test for `accumulate_item_timers/1` - -This might feel like we are working in reverse, -that's because we _are_! -We are working _back_ from our stated goal -of accumulating all the `timer` for a given `item` -so that we can display a _single_ elapsed time -when an `item` has had more than one timer. - -Open the -`test/app/item_test.exs` -file and add the following block of test code: - -```elixir - describe "accumulate timers for a list of items #103" do - test "accumulate_item_timers/1 to display cumulative timer" do - # https://hexdocs.pm/elixir/1.13/NaiveDateTime.html#new/2 - # "Add" -7 seconds: https://hexdocs.pm/elixir/1.13/Time.html#add/3 - {:ok, seven_seconds_ago} = - NaiveDateTime.new(Date.utc_today, Time.add(Time.utc_now, -7)) - - # this is the "shape" of the data that items_with_timers/1 will return: - items_with_timers = [ - %{ - stop: nil, - id: 3, - start: nil, - text: "This item has no timers", - timer_id: nil - }, - %{ - stop: ~N[2022-07-17 11:18:10.000000], - id: 2, - start: ~N[2022-07-17 11:18:00.000000], - text: "Item #2 has one active (no end) and one complete timer should total 17sec", - timer_id: 3 - }, - %{ - stop: nil, - id: 2, - start: seven_seconds_ago, - text: "Item #2 has one active (no end) and one complete timer should total 17sec", - timer_id: 4 - }, - %{ - stop: ~N[2022-07-17 11:18:31.000000], - id: 1, - start: ~N[2022-07-17 11:18:26.000000], - text: "Item with 3 complete timers that should add up to 42 seconds elapsed", - timer_id: 2 - }, - %{ - stop: ~N[2022-07-17 11:18:24.000000], - id: 1, - start: ~N[2022-07-17 11:18:18.000000], - text: "Item with 3 complete timers that should add up to 42 seconds elapsed", - timer_id: 1 - }, - %{ - stop: ~N[2022-07-17 11:19:42.000000], - id: 1, - start: ~N[2022-07-17 11:19:11.000000], - text: "Item with 3 complete timers that should add up to 42 seconds elapsed", - timer_id: 5 - } - ] - - # The *interesting* timer is the *active* one (started seven_seconds_ago) ... - # The "hard" part to test in accumulating timers are the *active* ones ... - acc = Item.accumulate_item_timers(items_with_timers) - item_map = Map.new(acc, fn item -> {item.id, item} end) - item1 = Map.get(item_map, 1) - item2 = Map.get(item_map, 2) - item3 = Map.get(item_map, 3) - - # It's easy to calculate time elapsed for timers that have an stop: - assert NaiveDateTime.diff(item1.stop, item1.start) == 42 - # This is the fun one that we need to be 17 seconds: - assert NaiveDateTime.diff(NaiveDateTime.utc_now(), item2.start) == 17 - # The diff will always be 17 seconds because we control the start in the test data above. - # But we still get the function to calculate it so we know it works. - - # The 3rd item doesn't have any timers, it's the control: - assert item3.start == nil - end - end -``` - -This is a large test but most of it is the test data (`items_with_timers`) in the format we will be returning from -`items_with_timers/1` in the next section. - -With that test in place, we can write the function. - -## 5.2 Implement the `accummulate_item_timers/1` function - -Open the -`lib/app/item.ex` -file and add the following function: - -```elixir -@doc """ - `accumulate_item_timers/1` aggregates the elapsed time - for all the timers associated with an item - and then subtracs that time from the start value of the *current* active timer. - This is done to create the appearance that a single timer is being started/stopped - when in fact there are multiple timers in the backend. - For MVP we *could* have just had a single timer ... - and given the "ugliness" of this code, I wish I had done that!! - But the "USP" of our product [IMO] is that - we can track the completion of a task across multiple work sessions. - And having multiple timers is the *only* way to achieve that. - - If you can think of a better way of achieving the same result, - please share: https://github.com/dwyl/app-mvp-phoenix/issues/103 - This function *relies* on the list of items being ordered by timer_id ASC - because it "pops" the last timer and ignores it to avoid double-counting. - """ - def accumulate_item_timers(items_with_timers) do - # e.g: %{0 => 0, 1 => 6, 2 => 5, 3 => 24, 4 => 7} - timer_id_diff_map = map_timer_diff(items_with_timers) - - # e.g: %{1 => [2, 1], 2 => [4, 3], 3 => []} - item_id_timer_id_map = Map.new(items_with_timers, fn i -> - { i.id, Enum.map(items_with_timers, fn it -> - if i.id == it.id, do: it.timer_id, else: nil - end) - # stackoverflow.com/questions/46339815/remove-nil-from-list - |> Enum.reject(&is_nil/1) - } - end) - - # this one is "wasteful" but I can't think of how to simplify it ... - item_id_timer_diff_map = Map.new(items_with_timers, fn item -> - timer_id_list = Map.get(item_id_timer_id_map, item.id, [0]) - # Remove last item from list before summing to avoid double-counting - {_, timer_id_list} = List.pop_at(timer_id_list, -1) - - { item.id, Enum.reduce(timer_id_list, 0, fn timer_id, acc -> - Map.get(timer_id_diff_map, timer_id) + acc - end) - } - end) - - # creates a nested map: %{ item.id: %{id: 1, text: "my item", etc.}} - Map.new(items_with_timers, fn item -> - time_elapsed = Map.get(item_id_timer_diff_map, item.id) - start = if is_nil(item.start), do: nil, - else: NaiveDateTime.add(item.start, -time_elapsed) - - { item.id, %{item | start: start}} - end) - # Return the list of items without duplicates and only the last/active timer: - |> Map.values() - # Sort list by item.id descending (ordered by timer_id ASC above) so newest item first: - |> Enum.sort_by(fn(i) -> i.id end, :desc) - end -``` - -There's no getting around this, -the function is huge and not very pretty. -But hopefully the comments clarify it. - -If anything is unclear, we're very happy to expand/explain. -We're also _very_ happy for anyone `else` to refactor it! -[Please open an issue](https://github.com/dwyl/app-mvp/issues/) -so we can discuss. πŸ™ - -## 5.3 Test for `items_with_timers/1` - -Open the -`test/app/item_test.exs` -file and the following test to the bottom: - -```elixir - test "Item.items_with_timers/1 returns a list of items with timers" do - {:ok, item1} = Item.create_item(@valid_attrs) - {:ok, item2} = Item.create_item(@valid_attrs) - assert Item.get_item!(item1.id).text == item1.text - - started = NaiveDateTime.utc_now() - - {:ok, timer1} = - Timer.start(%{item_id: item1.id, person_id: 1, start: started}) - {:ok, _timer2} = - Timer.start(%{item_id: item2.id, person_id: 1, start: started}) - - assert NaiveDateTime.diff(timer1.start, started) == 0 - - # list items with timers: - item_timers = Item.items_with_timers(1) - assert length(item_timers) > 0 - end -``` - -## 5.4 Implement `items_with_timers/1` - -Open the -`lib/app/item.ex` -file and add the following code to the bottom: - -```elixir -@doc """ - `items_with_timers/1` Returns a List of items with the latest associated timers. - - ## Examples - - iex> items_with_timers() - [ - %{text: "hello", person_id: 1, status: 2, start: 2022-07-14 09:35:18}, - %{text: "world", person_id: 2, status: 7, start: 2022-07-15 04:20:42} - ] - """ - # - def items_with_timers(person_id \\ 0) do - sql = """ - SELECT i.id, i.text, i.status, i.person_id, t.start, t.stop, t.id as timer_id FROM items i - FULL JOIN timers as t ON t.item_id = i.id - WHERE i.person_id = $1 AND i.status IS NOT NULL AND i.status != 6 - ORDER BY timer_id ASC; - """ - - Ecto.Adapters.SQL.query!(Repo, sql, [person_id]) - |> map_columns_to_values() - |> accumulate_item_timers() - end - - - @doc """ - `map_columns_to_values/1` takes an Ecto SQL query result - which has the List of columns and rows separate - and returns a List of Maps where the keys are the column names and values the data. - - Sadly, Ecto returns rows without column keys so we have to map them manually: - ref: https://groups.google.com/g/elixir-ecto/c/0cubhSd3QS0/m/DLdQsFrcBAAJ - """ - def map_columns_to_values(res) do - Enum.map(res.rows, fn(row) -> - Enum.zip(res.columns, row) - |> Map.new |> AtomicMap.convert() - end) - end - - @doc """ - `map_timer_diff/1` transforms a list of items_with_timers - into a flat map where the key is the timer_id and the value is the difference - between timer.stop and timer.start - If there is no active timer return {0, 0}. - If there is no timer.stop return Now - timer.start - - ## Examples - - iex> list = [ - %{ stop: nil, id: 3, start: nil, timer_id: nil }, - %{ stop: ~N[2022-07-17 11:18:24], id: 1, start: ~N[2022-07-17 11:18:18], timer_id: 1 }, - %{ stop: ~N[2022-07-17 11:18:31], id: 1, start: ~N[2022-07-17 11:18:26], timer_id: 2 }, - %{ stop: ~N[2022-07-17 11:18:24], id: 2, start: ~N[2022-07-17 11:18:00], timer_id: 3 }, - %{ stop: nil, id: 2, start: seven_seconds_ago, timer_id: 4 } - ] - iex> map_timer_diff(list) - %{0 => 0, 1 => 6, 2 => 5, 3 => 24, 4 => 7} - """ - def map_timer_diff(list) do - Map.new(list, fn item -> - if is_nil(item.timer_id) do - # item without any active timer - { 0, 0} - else - { item.timer_id, timer_diff(item)} - end - end) - end - - @doc """ - `timer_diff/1` calculates the difference between timer.stop and timer.start - If there is no active timer OR timer has not ended return 0. - The reasoning is: an *active* timer (no end) does not have to - be subtracted from the timer.start in the UI ... - Again, DRAGONS! - """ - def timer_diff(timer) do - # ignore timers that have not ended (current timer is factored in the UI!) - if is_nil(timer.stop) do - 0 - else - NaiveDateTime.diff(timer.stop, timer.start) - end - end -``` - -Once again, there is quite a lot going on here. -We have broken down the functions into chunks -and added inline comments to clarify the code. -But again, if anything is unclear please let us know!! - - -# 6. Add Authentication +# X. Add Authentication This section borrows heavily from: [dwyl/phoenix-liveview-chat-example](https://github.com/dwyl/phoenix-liveview-chat-example#12-authentication) -## 6.1 Add `auth_plug` to `deps` +## X.1 Add `auth_plug` to `deps` Open the `mix.exs` file and add `auth_plug` to the `deps` section: @@ -1399,14 +902,14 @@ run: mix deps.get ``` -## 6.2 Get your `AUTH_API_KEY` +## X.2 Get your `AUTH_API_KEY` Follow the steps in the [docs](https://github.com/dwyl/auth_plug#2-get-your-auth_api_key-) to get your `AUTH_API_KEY` environment variable. (1 minute) -## 6.2 Create Auth Controller +## X.3 Create Auth Controller Create a new file with the path: `lib/app_web/controllers/auth_controller.ex` @@ -1450,7 +953,7 @@ defmodule AppWeb.AuthController do end ``` -# 7. Create `LiveView` Functions +# Y. Create `LiveView` Functions _Finally_ we have all the "CRUD" functions we're going to need we can focus on the `LiveView` code that will be the actual UI/UX! @@ -1458,7 +961,7 @@ we can focus on the `LiveView` code that will be the actual UI/UX! ## 7.1 Write `LiveView` Tests Opent the -`test/app_web/live/app_live_test.exs` +`test/app_web/live/app_live_test.exs` file and replace the contents with the following test code: ```elixir diff --git a/lib/app/github.ex b/lib/app/github.ex index 33072d6..5f607a3 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -10,7 +10,7 @@ defmodule App.GitHub do @doc """ - Returns the GitHub user profile data. + Returns the GitHub repository data. """ def repository(owner, reponame) do Logger.info "Fetching repository #{owner}/#{reponame}" @@ -26,6 +26,5 @@ defmodule App.GitHub do Logger.info "Fetching user #{username}" {_status, data, _res} = Tentacat.Users.find @client, username data |> Useful.atomize_map_keys() - end end From 3ace5be4d94511d567bffb1e4cac5edfb6f8898c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 25 Dec 2024 08:44:22 +0000 Subject: [PATCH 06/58] add real test assertions to test/app/github_test.exs for #17 --- BUILDIT.md | 44 +++++++++++++++------------------------- lib/app/github.ex | 1 + test/app/github_test.exs | 11 +++++----- 3 files changed, 22 insertions(+), 34 deletions(-) diff --git a/BUILDIT.md b/BUILDIT.md index aad1715..b552c98 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -43,27 +43,12 @@ where (_hopefully_) it will all be clear. > please open an issue: > [dwyl/who/issues](https://github.com/dwyl/who/issues) -At the end of each step, -remember to run the tests: - -```sh -mix test -``` - -This will help you keep track of where you are -and retrace your steps if something is not working as expected. - -We suggest keeping two terminal tabs/windows running;
-one for the server `mix phx.server` and the other for the **tests**.
-That way you can also see the UI as you progress. - -With that in place, let's get building! - [Build Log πŸ‘©πŸ»β€πŸ’»](#build-log-) - [0. Prerequisites: _Before_ You Start](#0-prerequisites-before-you-start) - [1. Create a New `Phoenix` App](#1-create-a-new-phoenix-app) - [1.1 Run the `Phoenix` App](#11-run-the-phoenix-app) - - [1.2 Run the tests:](#12-run-the-tests) + - [1.2 Run the tests](#12-run-the-tests) - [Test Coverage? ](#test-coverage-) - [1.3 Setup `Tailwind`](#13-setup-tailwind) - [1.4 Setup `LiveView`](#14-setup-liveview) @@ -109,6 +94,7 @@ make sure you have `Phoenix` and `Postgres` installed, see how at: [dwyl/phoenix#how](https://github.com/dwyl/phoenix-chat-example?tab=readme-ov-file#how) +With everything installed & running, let's get building! πŸ‘·πŸ»β€β™€οΈ # 1. Create a New `Phoenix` App @@ -123,15 +109,13 @@ mix phx.new app --no-mailer --no-dashboard --no-gettext When asked to install the dependencies, type `Y` and `[Enter]` (_to install everything_). -The MVP won't +The `Who` App won't send emails, -display dashboards +display dashboards or translate to other languages (sorry).
-_All_ of those things -will be in the _main_ -[dwyl/**app**](https://github.com/dwyl/app).
-We're excluding them here +We can add `i18n` _later_. +We're excluding for now to reduce complexity/dependencies. ## 1.1 Run the `Phoenix` App @@ -163,11 +147,17 @@ You should see something similar to the following ![phoenix-default-homepage](https://user-images.githubusercontent.com/194400/174807257-34120dc5-723e-4b2c-9e8e-4b6f3aefca14.png) +## 1.2 Run the tests -## 1.2 Run the tests: +Run the tests with the command: -To run the tests with +```sh +mix test +``` +> **Note**: we recommend keeping _two_ terminal tabs/windows running;
+one for the server `mix phx.server` and the other for the **tests**.
+That way you can also see the UI as you progress. You should see output similar to: @@ -180,14 +170,12 @@ Finished in 0.1 seconds (0.07s async, 0.07s sync) That tells us everything is working as expected. πŸš€ - ### Test Coverage? [![codecov.io](https://img.shields.io/codecov/c/github/dwyl/who/main.svg?style=flat-square)](http://codecov.io/github/dwyl/who?branch=main) If you prefer to see **test coverage** - we certainly do - -then you will need to add a few lines to the +then you will need to add a few lines to the [`mix.exs`](https://github.com/dwyl/who/blob/main/mix.exs) -file and -create a +file and create a [`coveralls.json`](https://github.com/dwyl/who/blob/main/coveralls.json) file to exclude `Phoenix` files from `excoveralls` checking. Add alias (shortcuts) in `mix.exs` `defp aliases do` list. diff --git a/lib/app/github.ex b/lib/app/github.ex index 5f607a3..4edfa95 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -27,4 +27,5 @@ defmodule App.GitHub do {_status, data, _res} = Tentacat.Users.find @client, username data |> Useful.atomize_map_keys() end + end diff --git a/test/app/github_test.exs b/test/app/github_test.exs index 6d15aea..2767f34 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -5,14 +5,13 @@ defmodule App.GitHubTest do test "App.GitHub.repository/1" do owner = "dwyl" reponame = "start-here" - - GitHub.repository(owner, reponame) # |> IO.inspect - # assert html_response(conn, 200) =~ "LiveView App Page" + repo = GitHub.repository(owner, reponame) # |> dbg + assert repo.stargazers_count > 1700 end test "App.GitHub.user/1" do - user = "iteles" - GitHub.user(user) # |> IO.inspect - # assert html_response(conn, 200) =~ "LiveView App Page" + username = "iteles" + user = GitHub.user(username) # |> dbg + assert user.public_repos > 30 end end From 45e283f591a4586cf86719af3bc02fa3f2b15317 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 26 Dec 2024 07:31:36 +0000 Subject: [PATCH 07/58] config: deserialization_options: [keys: :atoms] --- config/config.exs | 3 ++- lib/app/github.ex | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/config.exs b/config/config.exs index da7b470..283471e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -55,7 +55,8 @@ config :joken, default_signer: System.get_env("SECRET_KEY_BASE") # api_key: System.get_env("AUTH_API_KEY") config :tentacat, - access_token: System.get_env("GH_PERSONAL_ACCESS_TOKEN") + access_token: System.get_env("GH_PERSONAL_ACCESS_TOKEN"), + deserialization_options: [keys: :atoms] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/lib/app/github.ex b/lib/app/github.ex index 4edfa95..17e79ba 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -16,7 +16,7 @@ defmodule App.GitHub do Logger.info "Fetching repository #{owner}/#{reponame}" {_status, data, _res} = Tentacat.Repositories.repo_get(@client, owner, reponame) - data |> Useful.atomize_map_keys() + data end @doc """ @@ -25,7 +25,7 @@ defmodule App.GitHub do def user(username) do Logger.info "Fetching user #{username}" {_status, data, _res} = Tentacat.Users.find @client, username - data |> Useful.atomize_map_keys() + data end end From 4efa7eade247efd957e71ecbb408d9a0693d727c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 26 Dec 2024 07:56:49 +0000 Subject: [PATCH 08/58] App.GitHub.org_user_list/1 now returns *all* 521 users in a single API request #17 --- lib/app/github.ex | 9 +++++++++ test/app/github_test.exs | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/lib/app/github.ex b/lib/app/github.ex index 17e79ba..6f19e42 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -28,4 +28,13 @@ defmodule App.GitHub do data end + @doc """ + `org_user_list/1` Returns the list of GitHub users for an org. + """ + def org_user_list(orgname) do + Logger.info "Fetching org user list for #{orgname}" + {_status, data, _res} = + Tentacat.Organizations.Members.list(@client, orgname) + data + end end diff --git a/test/app/github_test.exs b/test/app/github_test.exs index 2767f34..d105125 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -14,4 +14,10 @@ defmodule App.GitHubTest do user = GitHub.user(username) # |> dbg assert user.public_repos > 30 end + + test "App.org_user_list/1" do + orgname = "dwyl" + list = GitHub.org_user_list(orgname) + assert length(list) > 500 + end end From ab51741e4bef29de0b0e062872e9f80f422503e4 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 7 Jan 2025 10:49:03 +0000 Subject: [PATCH 09/58] data stream to client working! #17 --- lib/app/user.ex | 31 +++---- lib/app_web/live/app_live.ex | 83 ++++++++++++++++++- lib/app_web/live/app_live.html.heex | 26 +++++- lib/app_web/router.ex | 2 +- .../20221005213110_create_users.exs | 4 +- test/app/github_test.exs | 6 +- test/app/user_test.exs | 7 +- 7 files changed, 124 insertions(+), 35 deletions(-) diff --git a/lib/app/user.ex b/lib/app/user.ex index 57df4ee..65d0394 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -28,7 +28,7 @@ defmodule App.User do def changeset(user, attrs) do user |> cast(attrs, [:id, :login, :avatar_url, :name, :company, :bio, :blog, :location, :email, :created_at, :hireable, :two_factor_authentication, :public_repos, :followers, :following]) - |> validate_required([:id, :login, :avatar_url, :name, :created_at, :followers, :following]) + |> validate_required([:id, :login, :avatar_url, :created_at, :followers, :following]) end @doc """ @@ -40,27 +40,12 @@ defmodule App.User do |> Repo.insert(on_conflict: :replace_all, conflict_target: [:id]) end - # Envar.require_env_file(".env") - - def get_org_members_from_api(org_name) do - token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") - - client = Tentacat.Client.new(%{access_token: token}) - {200, data, _res} = Tentacat.Organizations.Members.list(client, org_name) - # dbg(data) - Useful.atomize_map_keys(data) - end - def get_user_from_api(username) do - token = Envar.get("GH_PERSONAL_ACCESS_TOKEN") - client = Tentacat.Client.new(%{access_token: token}) - {200, data, _res} = Tentacat.Users.find(client, username) - {:ok, entry} = Useful.atomize_map_keys(data) - |> dbg + {:ok, data} = App.GitHub.user(username) |> map_github_user_fields_to_table() |> create() - entry + data end # Next: get list of org members @@ -74,7 +59,7 @@ defmodule App.User do avatar_url: String.split(u.avatar_url, "?") |> List.first, bio: u.bio, blog: u.blog, - company: String.replace(u.company, "@", ""), + company: clean_company(u.company), created_at: u.created_at, email: u.email, followers: u.followers, @@ -89,5 +74,13 @@ defmodule App.User do } end + def clean_company(company) do + # avoid `String.replace(nil, "@", "", [])` error + if company == nil do + "" + else + String.replace(company, "@", "") + end + end end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 1800217..aaaf0d6 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -1,7 +1,84 @@ defmodule AppWeb.AppLive do use AppWeb, :live_view - def mount(_params, _session, socket) do - {:ok, socket} + @topic "live" + @img "https://avatars.githubusercontent.com/u/" + + def mount(_session, _params, socket) do + if connected?(socket) do + AppWeb.Endpoint.subscribe(@topic) # subscribe to the channel + end + p = %{id: 1, login: "Alex", avatar_url: "#{@img}1"} + {:ok, assign(socket, %{data: p})} + end + + def handle_event("inc", _value, socket) do + dbg(socket.assigns) + val = socket.assigns.data.id + 1 + p = %{id: val, login: "Alex #{val}", avatar_url: "#{@img}#{val}"} + new_state = assign(socket, %{data: p}) + broadcast("inc", new_state.assigns) + + {:noreply, new_state} + end + + def handle_event("dec", _, socket) do + val = socket.assigns.val - 1 + p = %{id: val, login: "Alex", avatar_url: "#{@img}#{val}"} + new_state = assign(socket, %{data: p}) + broadcast("dec", new_state.assigns) + {:noreply, new_state} + end + + def handle_event("sync", _value, socket) do + IO.inspect("sync") + sync(socket) + # val = socket.assigns.val + 1 + # p = %{login: "Alex", avatar_url: "#{@img}#{val}"} + # new_state = assign(socket, %{val: val, data: p}) + # broadcast("inc", new_state.assigns) + + # :timer.apply_interval(1000, IO, :puts, ["weeeee"]) + + {:noreply, socket} + end + + def handle_event("update", _value, socket) do + IO.inspect("- - - - - - - - - - - - - - - - - - - handle_event: update") + {:noreply, socket} + end + + # update `data` by broadcasting it as the profiles are crawled: + def sync(socket) do + + dbg(socket.assigns) + list = App.GitHub.org_user_list("dwyl") + # dbg(list) + # Iterate through the list of people + # Enum.map(list, Task.async( fn u -> + # dbg(u) + # end)) + list + |> Stream.with_index + |> Enum.map(fn {u, i} -> + IO.inspect("- - - Enum.map u.login: #{i}: #{u.login}") + data = App.User.get_user_from_api(u.login) + data = AuthPlug.Helpers.strip_struct_metadata(data) + new_state = assign(socket, %{data: data}) + Task.start(fn -> + :timer.sleep(300 + 100 * i) + AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) + end) + end) + + {:noreply, socket} + end + + def handle_info(msg, socket) do + {:noreply, assign(socket, data: msg.payload.data)} + end + + def broadcast(msg, assigns) do + AppWeb.Endpoint.broadcast_from(self(), @topic, msg, assigns) end -end \ No newline at end of file +end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index eb5be82..e838fde 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -1,4 +1,28 @@

hello david -

\ No newline at end of file + + +
+

+ The count is: <%= @data.id %>

+ +

+ + +

+
+ +

{ @data.login }

+ + + + + \ No newline at end of file diff --git a/lib/app_web/router.ex b/lib/app_web/router.ex index f371db5..1e6ef1b 100644 --- a/lib/app_web/router.ex +++ b/lib/app_web/router.ex @@ -16,7 +16,7 @@ defmodule AppWeb.Router do scope "/", AppWeb do pipe_through :browser - + live "/", AppLive end diff --git a/priv/repo/migrations/20221005213110_create_users.exs b/priv/repo/migrations/20221005213110_create_users.exs index 24297cb..49f362d 100644 --- a/priv/repo/migrations/20221005213110_create_users.exs +++ b/priv/repo/migrations/20221005213110_create_users.exs @@ -11,12 +11,12 @@ defmodule App.Repo.Migrations.CreateUsers do add :email, :string add :followers, :integer add :following, :integer - add :hireable, :boolean, default: false, null: false + add :hireable, :boolean, default: false add :location, :string add :login, :string add :name, :string add :public_repos, :integer - add :two_factor_authentication, :boolean, default: false, null: false + add :two_factor_authentication, :boolean, default: false timestamps() end diff --git a/test/app/github_test.exs b/test/app/github_test.exs index d105125..12079d2 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -15,9 +15,9 @@ defmodule App.GitHubTest do assert user.public_repos > 30 end - test "App.org_user_list/1" do - orgname = "dwyl" + test "App.GitHub.org_user_list/1" do + orgname = "ideaq" list = GitHub.org_user_list(orgname) - assert length(list) > 500 + assert length(list) > 2 end end diff --git a/test/app/user_test.exs b/test/app/user_test.exs index 1d09539..9b8ee1e 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -26,13 +26,8 @@ defmodule App.UserTest do assert inserted_user.name == user.name end - test "get_org_members_from_api/1" do - App.User.get_org_members_from_api("dwyl") # |> dbg - assert true == true - end - test "get_user_from_api/1" do - data = App.User.get_user_from_api("iteles") |> dbg + data = App.User.get_user_from_api("iteles") # |> dbg assert data.public_repos > 30 end end From ea12050c5107f97a92fe21b0b7a7891185aec58c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 7 Jan 2025 12:00:18 +0000 Subject: [PATCH 10/58] use card for displaying github profile #17 --- lib/app_web/live/app_live.ex | 24 +++------- lib/app_web/live/app_live.html.heex | 53 +++++++++++---------- lib/app_web/templates/layout/root.html.heex | 43 +++++++++++++++-- 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index aaaf0d6..5dd922d 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -8,7 +8,9 @@ defmodule AppWeb.AppLive do if connected?(socket) do AppWeb.Endpoint.subscribe(@topic) # subscribe to the channel end - p = %{id: 1, login: "Alex", avatar_url: "#{@img}1"} + p = %{id: 183617417, login: "Alex", + avatar_url: "#{@img}183617417", name: "Alexander", + bio: "Love learning", created_at: "2015", company: "dwyl"} {:ok, assign(socket, %{data: p})} end @@ -23,7 +25,7 @@ defmodule AppWeb.AppLive do end def handle_event("dec", _, socket) do - val = socket.assigns.val - 1 + val = socket.assigns.data.id - 1 p = %{id: val, login: "Alex", avatar_url: "#{@img}#{val}"} new_state = assign(socket, %{data: p}) broadcast("dec", new_state.assigns) @@ -31,33 +33,19 @@ defmodule AppWeb.AppLive do end def handle_event("sync", _value, socket) do - IO.inspect("sync") sync(socket) - # val = socket.assigns.val + 1 - # p = %{login: "Alex", avatar_url: "#{@img}#{val}"} - # new_state = assign(socket, %{val: val, data: p}) - # broadcast("inc", new_state.assigns) - - # :timer.apply_interval(1000, IO, :puts, ["weeeee"]) {:noreply, socket} end def handle_event("update", _value, socket) do - IO.inspect("- - - - - - - - - - - - - - - - - - - handle_event: update") {:noreply, socket} end # update `data` by broadcasting it as the profiles are crawled: def sync(socket) do - - dbg(socket.assigns) - list = App.GitHub.org_user_list("dwyl") - # dbg(list) - # Iterate through the list of people - # Enum.map(list, Task.async( fn u -> - # dbg(u) - # end)) + list = App.GitHub.org_user_list("ideaq") + # Iterate through the list of people and fetch profiles from API list |> Stream.with_index |> Enum.map(fn {u, i} -> diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index e838fde..99318f2 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -1,28 +1,31 @@ -

- hello david -

- -
-

- The count is: <%= @data.id %>

- -

- - -

-
- -

{ @data.login }

- - - - \ No newline at end of file + + + +
+
+
+
+ + {@data.name} +
+

@{@data.login}

+

{@data.name}

+

{@data.bio}

+
    +
  • + Company + + {@data.company} +
    • +
  • + Joined On + {@data.created_at} +
    • +
+
+
+
\ No newline at end of file diff --git a/lib/app_web/templates/layout/root.html.heex b/lib/app_web/templates/layout/root.html.heex index 7038850..c0d5656 100644 --- a/lib/app_web/templates/layout/root.html.heex +++ b/lib/app_web/templates/layout/root.html.heex @@ -13,11 +13,44 @@ -
-
-

App MVP Phoenix

-
-
+ + + <%= @inner_content %> From f3c35b06e5f8315285a6a21e8ca235c039e70979 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 7 Jan 2025 12:01:39 +0000 Subject: [PATCH 11/58] fix failing test --- test/app_web/live/app_live_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/app_web/live/app_live_test.exs b/test/app_web/live/app_live_test.exs index 37ef70f..d01011d 100644 --- a/test/app_web/live/app_live_test.exs +++ b/test/app_web/live/app_live_test.exs @@ -3,6 +3,6 @@ defmodule AppWeb.AppLiveTest do test "GET /", %{conn: conn} do conn = get(conn, "/") - assert html_response(conn, 200) =~ "hello david" + assert html_response(conn, 200) =~ "sync" end end From eca058627b356664f4dd0bb0f35f56e33371e9e2 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 13 Jan 2025 09:12:19 +0000 Subject: [PATCH 12/58] invoke Useful.truncate/3 to restrict bio length see: https://github.com/dwyl/useful/issues/77#issuecomment-2586559239 --- lib/app_web/live/app_live.ex | 23 ++++++++++++++++------- lib/app_web/live/app_live.html.heex | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 5dd922d..7359b63 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -9,8 +9,9 @@ defmodule AppWeb.AppLive do AppWeb.Endpoint.subscribe(@topic) # subscribe to the channel end p = %{id: 183617417, login: "Alex", - avatar_url: "#{@img}183617417", name: "Alexander", - bio: "Love learning", created_at: "2015", company: "dwyl"} + avatar_url: "#{@img}128895421", name: "Alexander the Greatest", + bio: "Love learning how to code with my crew of cool cats!", + created_at: "2010-02-02T08:44:49Z", company: "dwyl"} {:ok, assign(socket, %{data: p})} end @@ -46,15 +47,14 @@ defmodule AppWeb.AppLive do def sync(socket) do list = App.GitHub.org_user_list("ideaq") # Iterate through the list of people and fetch profiles from API - list - |> Stream.with_index - |> Enum.map(fn {u, i} -> - IO.inspect("- - - Enum.map u.login: #{i}: #{u.login}") + Stream.with_index(list) + |> Enum.map(fn {u, index} -> + # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") data = App.User.get_user_from_api(u.login) data = AuthPlug.Helpers.strip_struct_metadata(data) new_state = assign(socket, %{data: data}) Task.start(fn -> - :timer.sleep(300 + 100 * i) + :timer.sleep(300 + 100 * index) AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) end) end) @@ -69,4 +69,13 @@ defmodule AppWeb.AppLive do def broadcast(msg, assigns) do AppWeb.Endpoint.broadcast_from(self(), @topic, msg, assigns) end + + # Template Helper Functions + def short_date(date) do + String.split(date, "T") |> List.first + end + + def truncate_bio(bio) do + Useful.truncate(bio, 29, " ...") + end end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index 99318f2..cabf1e3 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -14,7 +14,7 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round

@{@data.login}

{@data.name}

-

{@data.bio}

+

{truncate_bio(@data.bio)}

  • Company @@ -23,7 +23,7 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round
  • Joined On - {@data.created_at} + {short_date(@data.created_at)}
From 5704db0db094d60ba4c4fe66852fcf55620b08bc Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 14 Jan 2025 17:07:32 +0000 Subject: [PATCH 13/58] ignore the 404 for now ... #216 --- lib/app/user.ex | 42 ++++++++++++++++++++++++++++++++++- lib/app_web/live/app_live.ex | 43 ++++++++++++++---------------------- test/app/user_test.exs | 5 +++++ 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/lib/app/user.ex b/lib/app/user.ex index 65d0394..9bc1f2a 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -28,7 +28,7 @@ defmodule App.User do def changeset(user, attrs) do user |> cast(attrs, [:id, :login, :avatar_url, :name, :company, :bio, :blog, :location, :email, :created_at, :hireable, :two_factor_authentication, :public_repos, :followers, :following]) - |> validate_required([:id, :login, :avatar_url, :created_at, :followers, :following]) + |> validate_required([:id, :login, :avatar_url]) end @doc """ @@ -41,7 +41,27 @@ defmodule App.User do end def get_user_from_api(username) do + + + # case App.GitHub.user(user.login) do + # {:ok, data} -> + # cols = res.columns + # [first_row | _] = res.rows + # [new_id, validation_token, auth_token, success, message] = first_row + # {:ok, %RegistrationResult{ + # success: success, + # message: message, + # new_id: new_id, + # authentication_token: auth_token, + # validation_token: validation_token + # }} + + # {:error, err} -> + + # end + {:ok, data} = App.GitHub.user(username) + |> dbg() |> map_github_user_fields_to_table() |> create() @@ -83,4 +103,24 @@ defmodule App.User do end end + # create function that returns dummy user data + def dummy_data(u \\ %{}) do + Map.merge(%{ + id: :rand.uniform(1_000_000_000) + 1_000_000_000, + avatar_url: "https://avatars.githubusercontent.com/u/10137", + bio: "", + blog: "", + company: "good", + created_at: "", + email: "", + followers: 0, + following: 0, + hireable: false, + location: "", + login: "al3x", + name: "Lex", + public_repos: 0, + two_factor_authentication: false + }, u) + end end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 7359b63..2339b32 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -15,24 +15,6 @@ defmodule AppWeb.AppLive do {:ok, assign(socket, %{data: p})} end - def handle_event("inc", _value, socket) do - dbg(socket.assigns) - val = socket.assigns.data.id + 1 - p = %{id: val, login: "Alex #{val}", avatar_url: "#{@img}#{val}"} - new_state = assign(socket, %{data: p}) - broadcast("inc", new_state.assigns) - - {:noreply, new_state} - end - - def handle_event("dec", _, socket) do - val = socket.assigns.data.id - 1 - p = %{id: val, login: "Alex", avatar_url: "#{@img}#{val}"} - new_state = assign(socket, %{data: p}) - broadcast("dec", new_state.assigns) - {:noreply, new_state} - end - def handle_event("sync", _value, socket) do sync(socket) @@ -45,18 +27,25 @@ defmodule AppWeb.AppLive do # update `data` by broadcasting it as the profiles are crawled: def sync(socket) do - list = App.GitHub.org_user_list("ideaq") + list = App.GitHub.org_user_list("dwyl") # Iterate through the list of people and fetch profiles from API Stream.with_index(list) |> Enum.map(fn {u, index} -> - # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") - data = App.User.get_user_from_api(u.login) - data = AuthPlug.Helpers.strip_struct_metadata(data) - new_state = assign(socket, %{data: data}) - Task.start(fn -> - :timer.sleep(300 + 100 * index) - AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) - end) + # don't have time to waste on this right now ... + if u.login == "kittenking" do + IO.inspect(" - - - - - - - - - - - - - - - - - kittenking: ") + dbg(u) + IO.inspect(" - - - - - - - - - - - - - - - - - - - - - - - ") + else + # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") + data = App.User.get_user_from_api(u.login) + data = AuthPlug.Helpers.strip_struct_metadata(data) + new_state = assign(socket, %{data: data}) + Task.start(fn -> + :timer.sleep(300 + 100 * index) + AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) + end) + end end) {:noreply, socket} diff --git a/test/app/user_test.exs b/test/app/user_test.exs index 9b8ee1e..1c9ac11 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -30,6 +30,11 @@ defmodule App.UserTest do data = App.User.get_user_from_api("iteles") # |> dbg assert data.public_repos > 30 end + + test "dummy_data/0" do + data = App.User.dummy_data(%{id: 42}) + assert data.company == "good" + end end %{ From e3b41a45ae320a01b8faba4172ae6431c3ddfb32 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 15 Jan 2025 03:26:52 +0000 Subject: [PATCH 14/58] address 404 error #216 --- lib/app/github.ex | 2 +- lib/app/user.ex | 40 +++++++++++++++--------------------- lib/app_web/live/app_live.ex | 23 ++++++++------------- test/app/github_test.exs | 9 ++++++++ test/app/user_test.exs | 17 ++++++++++++++- 5 files changed, 50 insertions(+), 41 deletions(-) diff --git a/lib/app/github.ex b/lib/app/github.ex index 6f19e42..5a6cf90 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -24,7 +24,7 @@ defmodule App.GitHub do """ def user(username) do Logger.info "Fetching user #{username}" - {_status, data, _res} = Tentacat.Users.find @client, username + {_status, data, _res} = Tentacat.Users.find(@client, username) data end diff --git a/lib/app/user.ex b/lib/app/user.ex index 9bc1f2a..314f2b7 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -40,32 +40,24 @@ defmodule App.User do |> Repo.insert(on_conflict: :replace_all, conflict_target: [:id]) end - def get_user_from_api(username) do + # `user` map must include the `id` and `login` fields + def get_user_from_api(user) do + data = App.GitHub.user(user.login) + # prolly wouldn't do this in a "real" app ... feel free to refactor. + if Map.has_key?(data, :status) && data.status == "404" do + # IO.inspect(" - - - - - - - - - - - - - - - - - #{user.login}") + # dbg(user) + # IO.inspect(" - - - - - - - - - - - - - - - - - - - - - - - ") + {:ok, user} = dummy_data(user) |> create() + user + else + {:ok, user} = + map_github_user_fields_to_table(data) + |> create() - # case App.GitHub.user(user.login) do - # {:ok, data} -> - # cols = res.columns - # [first_row | _] = res.rows - # [new_id, validation_token, auth_token, success, message] = first_row - # {:ok, %RegistrationResult{ - # success: success, - # message: message, - # new_id: new_id, - # authentication_token: auth_token, - # validation_token: validation_token - # }} - - # {:error, err} -> - - # end - - {:ok, data} = App.GitHub.user(username) - |> dbg() - |> map_github_user_fields_to_table() - |> create() - - data + user + end end # Next: get list of org members diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 2339b32..eaa68c3 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -31,21 +31,14 @@ defmodule AppWeb.AppLive do # Iterate through the list of people and fetch profiles from API Stream.with_index(list) |> Enum.map(fn {u, index} -> - # don't have time to waste on this right now ... - if u.login == "kittenking" do - IO.inspect(" - - - - - - - - - - - - - - - - - kittenking: ") - dbg(u) - IO.inspect(" - - - - - - - - - - - - - - - - - - - - - - - ") - else - # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") - data = App.User.get_user_from_api(u.login) - data = AuthPlug.Helpers.strip_struct_metadata(data) - new_state = assign(socket, %{data: data}) - Task.start(fn -> - :timer.sleep(300 + 100 * index) - AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) - end) - end + # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") + data = App.User.get_user_from_api(u) + data = AuthPlug.Helpers.strip_struct_metadata(data) + new_state = assign(socket, %{data: data}) + Task.start(fn -> + :timer.sleep(300 + 100 * index) + AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) + end) end) {:noreply, socket} diff --git a/test/app/github_test.exs b/test/app/github_test.exs index 12079d2..b2a9051 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -20,4 +20,13 @@ defmodule App.GitHubTest do list = GitHub.org_user_list(orgname) assert length(list) > 2 end + + test "App.GitHub.user/1 known 404 (unhappy path)" do + username ="kittenking" + data = App.GitHub.user(username) + assert data.status == "404" + end + + + end diff --git a/test/app/user_test.exs b/test/app/user_test.exs index 1c9ac11..0d15841 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -27,10 +27,25 @@ defmodule App.UserTest do end test "get_user_from_api/1" do - data = App.User.get_user_from_api("iteles") # |> dbg + data = App.User.get_user_from_api(%{login: "iteles"}) # |> dbg assert data.public_repos > 30 end + test "get_user_from_api/1 unhappy path (kittenking)" do + user = %{ + id: 53072918, + type: "User", + url: "https://api.github.com/users/kittenking", + avatar_url: "https://avatars.githubusercontent.com/u/53072918?v=4", + login: "kittenking", + node_id: "MDQ6VXNlcjUzMDcyOTE4", + user_view_type: "public", + site_admin: false + } + data = App.User.get_user_from_api(user) + assert data.id == user.id + end + test "dummy_data/0" do data = App.User.dummy_data(%{id: 42}) assert data.company == "good" From b4846e0cdc218406280e4a6662da5fabf214e641 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 15 Jan 2025 06:25:08 +0000 Subject: [PATCH 15/58] add tests and cut untested code. everything still works. #17 --- .env_sample | 4 +++- lib/app_web/live/app_live.ex | 36 ++++++++++++++++------------- lib/app_web/live/app_live.html.heex | 20 ++++++++++++---- test/app/github_test.exs | 3 --- test/app_web/live/app_live_test.exs | 24 +++++++++++++++++++ 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/.env_sample b/.env_sample index edb48a2..ebab2bf 100644 --- a/.env_sample +++ b/.env_sample @@ -3,4 +3,6 @@ export SECRET_KEY_BASE=2PzB7PPnpuLsbWmWtXpGyI+kfSQSQ1zUW2Atz/+8PdZuSEJzHgzGnJWV3 export AUTH_API_KEY=YTsV7fG5mZ2KRWmvE3u431sZYsaZhhC8oqvQSDg85VnqMQXSDEBjh/YTsV7TBnHp1yxy2LLxBZYyVBrTYPtiKjLbApKiFkva3YQ8rrGgYeV/authdemo.fly.dev # https://github.com/settings/tokens/new -export GH_PERSONAL_ACCESS_TOKEN=YourTokenHere \ No newline at end of file +export GH_PERSONAL_ACCESS_TOKEN=YourTokenHere + +export ORG_NAME=dwyl \ No newline at end of file diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index eaa68c3..fbae858 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -8,32 +8,44 @@ defmodule AppWeb.AppLive do if connected?(socket) do AppWeb.Endpoint.subscribe(@topic) # subscribe to the channel end + p = %{id: 183617417, login: "Alex", avatar_url: "#{@img}128895421", name: "Alexander the Greatest", bio: "Love learning how to code with my crew of cool cats!", - created_at: "2010-02-02T08:44:49Z", company: "dwyl"} + created_at: "2010-02-02T08:44:49Z", company: "ideaq"} {:ok, assign(socket, %{data: p})} end - def handle_event("sync", _value, socket) do - sync(socket) + def handle_event("sync", value, socket) do + # IO.inspect("handle_event:sync - - - - -") + org = socket.assigns.data.company + override = if value && Map.has_key?(value, "org") do + # dbg(value) + Map.get(value, "org") + end + + sync(socket, override || org) {:noreply, socket} end - def handle_event("update", _value, socket) do - {:noreply, socket} + # def handle_event("update", _value, socket) do + # {:noreply, socket} + # end + + def handle_info(msg, socket) do + {:noreply, assign(socket, data: msg.payload.data)} end # update `data` by broadcasting it as the profiles are crawled: - def sync(socket) do - list = App.GitHub.org_user_list("dwyl") + def sync(socket, org) do + list = App.GitHub.org_user_list(org) # Iterate through the list of people and fetch profiles from API Stream.with_index(list) |> Enum.map(fn {u, index} -> # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") data = App.User.get_user_from_api(u) - data = AuthPlug.Helpers.strip_struct_metadata(data) + |> AuthPlug.Helpers.strip_struct_metadata() new_state = assign(socket, %{data: data}) Task.start(fn -> :timer.sleep(300 + 100 * index) @@ -44,14 +56,6 @@ defmodule AppWeb.AppLive do {:noreply, socket} end - def handle_info(msg, socket) do - {:noreply, assign(socket, data: msg.payload.data)} - end - - def broadcast(msg, assigns) do - AppWeb.Endpoint.broadcast_from(self(), @topic, msg, assigns) - end - # Template Helper Functions def short_date(date) do String.split(date, "T") |> List.first diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index cabf1e3..82f639c 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -3,8 +3,8 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round Sync -
+

Newest person:

@@ -12,10 +12,16 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round {@data.name}
-

@{@data.login}

-

{@data.name}

-

{truncate_bio(@data.bio)}

-
    +

    + @{@data.login} +

    +

    + {@data.name} +

    +

    + {truncate_bio(@data.bio)} +

    +
    • Company @@ -28,4 +34,8 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round
+
+ +
+

hello

\ No newline at end of file diff --git a/test/app/github_test.exs b/test/app/github_test.exs index b2a9051..ecd9c47 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -26,7 +26,4 @@ defmodule App.GitHubTest do data = App.GitHub.user(username) assert data.status == "404" end - - - end diff --git a/test/app_web/live/app_live_test.exs b/test/app_web/live/app_live_test.exs index d01011d..f4a99d2 100644 --- a/test/app_web/live/app_live_test.exs +++ b/test/app_web/live/app_live_test.exs @@ -1,8 +1,32 @@ defmodule AppWeb.AppLiveTest do use AppWeb.ConnCase + import Phoenix.LiveViewTest + alias AppWeb.AppLive test "GET /", %{conn: conn} do conn = get(conn, "/") assert html_response(conn, 200) =~ "sync" end + + test "default profile", %{conn: conn} do + {:ok, _view, html} = live(conn, "/") + assert html =~ "Alexander the Greatest" + end + + test "trigger sync", %{conn: conn} do + {:ok, view, _html} = live(conn, "/") + assert render_hook(view, :sync, %{org: "ideaq"}) + =~ "Love learning how to code" + end + + describe "template helper functions" do + test "short_date/1 shortens the date received from GitHub" do + assert AppLive.short_date("2010-02-02T08:44:49Z") == "2010-02-02" + end + + test "truncate_bio/1 truncates the bio to 29 chars" do + bio = "It was a bright cold day in April, and the clocks were striking 13" + assert AppLive.truncate_bio(bio) == "It was a bright cold day in ..." + end + end end From 2310d403caaaf3d342b2a431fdba06b244affe22 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 15 Jan 2025 07:45:58 +0000 Subject: [PATCH 16/58] render wall of faces #219 --- lib/app/user.ex | 47 +++++++++++++------------ lib/app_web/live/app_live.ex | 9 ++++- lib/app_web/live/app_live.html.heex | 6 ++-- test/app/user_test.exs | 54 ++++++++--------------------- 4 files changed, 51 insertions(+), 65 deletions(-) diff --git a/lib/app/user.ex b/lib/app/user.ex index 314f2b7..a14e8e1 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -1,7 +1,7 @@ defmodule App.User do use Ecto.Schema alias App.{Repo} - import Ecto.Changeset + import Ecto.{Changeset, Query} require Logger alias __MODULE__ @@ -43,7 +43,7 @@ defmodule App.User do # `user` map must include the `id` and `login` fields def get_user_from_api(user) do data = App.GitHub.user(user.login) - # prolly wouldn't do this in a "real" app ... feel free to refactor. + # Not super happy about this crude error handling ... feel free to refactor. if Map.has_key?(data, :status) && data.status == "404" do # IO.inspect(" - - - - - - - - - - - - - - - - - #{user.login}") # dbg(user) @@ -60,30 +60,12 @@ defmodule App.User do end end - # Next: get list of org members - # get user for each in the list - # map data to our table - # insert data - + # tidy data before insertion def map_github_user_fields_to_table(u) do - %{ - id: u.id, + Map.merge(u, %{ avatar_url: String.split(u.avatar_url, "?") |> List.first, - bio: u.bio, - blog: u.blog, company: clean_company(u.company), - created_at: u.created_at, - email: u.email, - followers: u.followers, - following: u.following, - hireable: u.hireable, - location: u.location, - login: u.login, - name: u.name, - public_repos: u.public_repos, - two_factor_authentication: false, - updated_at: u.updated_at - } + }) end def clean_company(company) do @@ -115,4 +97,23 @@ defmodule App.User do two_factor_authentication: false }, u) end + + # def list_users do + # User + # |> limit(20) + # |> order_by(desc: :inserted_at) + # |> Repo.all() + # end + + def list_users_avatars do + from(u in User, select: %{avatar_url: u.avatar_url}) + # |> limit(20) + |> order_by(desc: :inserted_at) + # |> distinct(true) + |> Repo.all() + # return a list of urls not a list of maps + |> Enum.reduce([], fn u, acc -> + [u.avatar_url | acc] + end) + end end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index fbae858..058246b 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -13,7 +13,10 @@ defmodule AppWeb.AppLive do avatar_url: "#{@img}128895421", name: "Alexander the Greatest", bio: "Love learning how to code with my crew of cool cats!", created_at: "2010-02-02T08:44:49Z", company: "ideaq"} - {:ok, assign(socket, %{data: p})} + # NEXT: prepend avatars to list ... + + + {:ok, assign(socket, %{data: p, avatars: App.User.list_users_avatars()})} end def handle_event("sync", value, socket) do @@ -64,4 +67,8 @@ defmodule AppWeb.AppLive do def truncate_bio(bio) do Useful.truncate(bio, 29, " ...") end + + def tiny_avatar(src) do + "#{src}?s=40" + end end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index 82f639c..e613ee7 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -36,6 +36,8 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round -
-

hello

+
+ + +
\ No newline at end of file diff --git a/test/app/user_test.exs b/test/app/user_test.exs index 0d15841..e9dbef9 100644 --- a/test/app/user_test.exs +++ b/test/app/user_test.exs @@ -1,9 +1,9 @@ defmodule App.UserTest do use App.DataCase - # alias App.User + alias App.User - test "App.User.create/1" do - user = %{ + def user do + %{ avatar_url: "https://avatars.githubusercontent.com/u/4185328?v=4", bio: "Co-founder @dwyl", blog: "https://www.twitter.com/iteles", @@ -22,6 +22,10 @@ defmodule App.UserTest do public_gists: 0, public_repos: 31 } + end + + test "App.User.create/1" do + user = user() assert {:ok, inserted_user} = App.User.create(user) assert inserted_user.name == user.name end @@ -32,6 +36,7 @@ defmodule App.UserTest do end test "get_user_from_api/1 unhappy path (kittenking)" do + # ref: https://github.com/dwyl/who/issues/216 user = %{ id: 53072918, type: "User", @@ -50,40 +55,11 @@ defmodule App.UserTest do data = App.User.dummy_data(%{id: 42}) assert data.company == "good" end -end -%{ - organizations_url: "https://api.github.com/users/iteles/orgs", - following: 80, - login: "iteles", - public_repos: 31, - received_events_url: "https://api.github.com/users/iteles/received_events", - bio: "Co-founder @dwyl \r\n", - user_view_type: "public", - company: "@dwyl", - gravatar_id: "", - twitter_username: nil, - following_url: "https://api.github.com/users/iteles/following{/other_user}", - created_at: "2013-04-17T21:10:06Z", - followers: 400, - site_admin: false, - blog: "http://www.twitter.com/iteles", - starred_url: "https://api.github.com/users/iteles/starred{/owner}{/repo}", - public_gists: 0, - hireable: true, - gists_url: "https://api.github.com/users/iteles/gists{/gist_id}", - events_url: "https://api.github.com/users/iteles/events{/privacy}", - followers_url: "https://api.github.com/users/iteles/followers", - node_id: "MDQ6VXNlcjQxODUzMjg=", - url: "https://api.github.com/users/iteles", - id: 4185328, - name: "Ines Teles Correia", - subscriptions_url: "https://api.github.com/users/iteles/subscriptions", - html_url: "https://github.com/iteles", - location: "London, UK", - type: "User", - avatar_url: "https://avatars.githubusercontent.com/u/4185328?v=4", - email: nil, - repos_url: "https://api.github.com/users/iteles/repos", - updated_at: "2024-08-05T22:59:09Z" -} + test "list_users_avatars/0" do + user = user() |> User.map_github_user_fields_to_table() + User.create(user) + list = App.User.list_users_avatars() + assert length(list) > 0 + end +end From 12c321222b284d9da49c2382dc67e7674222d0a1 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 16 Jan 2025 08:59:48 +0000 Subject: [PATCH 17/58] setup deps, files, functions & tests for image color extraction #221 --- lib/app/github.ex | 10 +++++++ lib/app/image.ex | 28 +++++++++++++++++++ lib/app/repository.ex | 14 +++++++++- lib/app/user.ex | 4 +-- lib/app_web/live/app_live.ex | 11 ++++++-- lib/app_web/live/app_live.html.heex | 4 +-- mix.exs | 3 ++ .../20221005213326_create_repositories.exs | 1 + test/app/github_test.exs | 6 ++++ test/app/image_test.exs | 17 +++++++++++ test/app/repository_test.exs | 6 ++++ test/app_web/live/app_live_test.exs | 5 ++++ 12 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 lib/app/image.ex create mode 100644 test/app/image_test.exs diff --git a/lib/app/github.ex b/lib/app/github.ex index 5a6cf90..285b6c9 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -19,6 +19,16 @@ defmodule App.GitHub do data end + @doc """ + Returns the list of GitHub repositories for an Org. + """ + def org_repos(owner) do + Logger.info "Fetching list of repositories for #{owner}" + {_status, data, _res} = + Tentacat.Repositories.list_orgs(@client, owner) + data + end + @doc """ `user/1` Returns the GitHub user profile data. """ diff --git a/lib/app/image.ex b/lib/app/image.ex new file mode 100644 index 0000000..f4ccf21 --- /dev/null +++ b/lib/app/image.ex @@ -0,0 +1,28 @@ +defmodule App.Image do + @moduledoc """ + Handles extracting color data from images. + Uses: https://github.com/elixir-image/image + """ + require Logger + :inets.start() + :ssl.start() + + @doc """ + Retrieve raw image data from a URL. + https://stackoverflow.com/questions/30267943/elixir-download-image-from-url + """ + def get_raw_image_data(imgurl) do + {:ok, resp} = :httpc.request(:get, {imgurl, []}, [], [body_format: :binary]) + {{_, 200, ~c"OK"}, _headers, body} = resp + + body + end + + @doc """ + Returns the predominant color for an image. + https://hexdocs.pm/image/Image.html#dominant_color/2 + """ + def extract_color(img_data) do + "blue" + end +end diff --git a/lib/app/repository.ex b/lib/app/repository.ex index 2740461..0418a0a 100644 --- a/lib/app/repository.ex +++ b/lib/app/repository.ex @@ -11,6 +11,7 @@ defmodule App.Repository do field :fork, :boolean, default: false field :forks_count, :integer field :full_name, :string + field :language, :string field :name, :string field :open_issues_count, :integer field :owner_id, :integer @@ -25,7 +26,9 @@ defmodule App.Repository do @doc false def changeset(repository, attrs) do repository - |> cast(attrs, [:id, :name, :full_name, :owner_id, :description, :fork, :forks_count, :watchers_count, :stargazers_count, :topics, :open_issues_count, :created_at, :pushed_at]) + |> cast(attrs, [:id, :name, :full_name, :language, :owner_id, :description, + :fork, :forks_count, :watchers_count, :stargazers_count, :topics, + :open_issues_count, :created_at, :pushed_at]) |> validate_required([:name, :full_name]) end @@ -38,4 +41,13 @@ defmodule App.Repository do |> Repo.insert() end + @doc """ + Get all repositories for an organization and insert them into DB. + """ + def get_org_repos(org) do + App.GitHub.org_repos(org) + |> Enum.map(fn repo -> + create(repo) + end) + end end diff --git a/lib/app/user.ex b/lib/app/user.ex index a14e8e1..646f8cd 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -106,14 +106,14 @@ defmodule App.User do # end def list_users_avatars do - from(u in User, select: %{avatar_url: u.avatar_url}) + from(u in User, select: %{id: u.id}) # |> limit(20) |> order_by(desc: :inserted_at) # |> distinct(true) |> Repo.all() # return a list of urls not a list of maps |> Enum.reduce([], fn u, acc -> - [u.avatar_url | acc] + [u.id | acc] end) end end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 058246b..f2dc260 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -16,7 +16,7 @@ defmodule AppWeb.AppLive do # NEXT: prepend avatars to list ... - {:ok, assign(socket, %{data: p, avatars: App.User.list_users_avatars()})} + {:ok, assign(socket, %{data: p, ids: App.User.list_users_avatars()})} end def handle_event("sync", value, socket) do @@ -42,6 +42,11 @@ defmodule AppWeb.AppLive do # update `data` by broadcasting it as the profiles are crawled: def sync(socket, org) do + # Get Repos: + Task.start(fn -> + App.Repository.get_org_repos(org) + end) + list = App.GitHub.org_user_list(org) # Iterate through the list of people and fetch profiles from API Stream.with_index(list) @@ -68,7 +73,7 @@ defmodule AppWeb.AppLive do Useful.truncate(bio, 29, " ...") end - def tiny_avatar(src) do - "#{src}?s=40" + def avatar(id) do + "https://avatars.githubusercontent.com/u/#{id}?s=30" end end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index e613ee7..1a3fc89 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -37,7 +37,7 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round
- - + +
\ No newline at end of file diff --git a/mix.exs b/mix.exs index 0aec3f1..0ac034d 100644 --- a/mix.exs +++ b/mix.exs @@ -69,6 +69,9 @@ defmodule App.MixProject do # JSON Parsing: https://hex.pm/packages/poison {:poison, "~> 6.0.0"}, + # Extract image data: https://github.com/elixir-image/image/ + {:image, "~> 0.37"}, + # Create docs on localhost by running "mix docs" {:ex_doc, "~> 0.37.1", only: :dev, runtime: false}, # Track test coverage diff --git a/priv/repo/migrations/20221005213326_create_repositories.exs b/priv/repo/migrations/20221005213326_create_repositories.exs index 3162ad6..1e55f54 100644 --- a/priv/repo/migrations/20221005213326_create_repositories.exs +++ b/priv/repo/migrations/20221005213326_create_repositories.exs @@ -8,6 +8,7 @@ defmodule App.Repo.Migrations.CreateRepositories do add :fork, :boolean, default: false, null: false add :forks_count, :integer add :full_name, :string + add :language, :string add :name, :string add :open_issues_count, :integer add :owner_id, :integer diff --git a/test/app/github_test.exs b/test/app/github_test.exs index ecd9c47..1350021 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -26,4 +26,10 @@ defmodule App.GitHubTest do data = App.GitHub.user(username) assert data.status == "404" end + + test "App.GitHub.org_repos/1 get repos for org" do + org = "ideaq" + list = App.GitHub.org_repos(org) |> dbg + assert length(list) > 2 + end end diff --git a/test/app/image_test.exs b/test/app/image_test.exs new file mode 100644 index 0000000..a79ac47 --- /dev/null +++ b/test/app/image_test.exs @@ -0,0 +1,17 @@ +defmodule App.ImageTest do + use ExUnit.Case + alias App.Image + + test "App.Image.get_raw_image_data/1" do + url = "https://avatars.githubusercontent.com/u/4185328" + img_data = Image.get_raw_image_data(url) # |> dbg + assert is_binary(img_data) + end + + test "App.Image.extract_color/1 retrieves color" do + url = "https://avatars.githubusercontent.com/u/4185328" + img_data = Image.get_raw_image_data(url) + Image.extract_color(img_data) + + end +end diff --git a/test/app/repository_test.exs b/test/app/repository_test.exs index 52809a0..fcf2ea1 100644 --- a/test/app/repository_test.exs +++ b/test/app/repository_test.exs @@ -22,4 +22,10 @@ defmodule App.RepositoryTest do assert {:ok, inserted_repo} = App.Repository.create(repo) assert inserted_repo.name == repo.name end + + test "App.Repository.get_org_repos/1" do + + App.Repository.get_org_repos("ideaq") # |> dbg + # assert inserted_repo.name == repo.name + end end diff --git a/test/app_web/live/app_live_test.exs b/test/app_web/live/app_live_test.exs index f4a99d2..1081002 100644 --- a/test/app_web/live/app_live_test.exs +++ b/test/app_web/live/app_live_test.exs @@ -28,5 +28,10 @@ defmodule AppWeb.AppLiveTest do bio = "It was a bright cold day in April, and the clocks were striking 13" assert AppLive.truncate_bio(bio) == "It was a bright cold day in ..." end + + test "avatar/1 prepares the avatar_url for displaying face wall" do + assert AppLive.avatar(1) == + "https://avatars.githubusercontent.com/u/1?s=30" + end end end From 44a273ca28abe2cc93649f04f3bba4f1e0dcd542 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 16 Jan 2025 09:00:11 +0000 Subject: [PATCH 18/58] add image & vix deps for #221 --- mix.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mix.lock b/mix.lock index c9c97b7..523fbf9 100644 --- a/mix.lock +++ b/mix.lock @@ -26,6 +26,7 @@ "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.3", "67b3d9fa8691b727317e0cc96b9b3093be00ee45419ffb221cdeee88e75d1360", [:mix], [{:mochiweb, "~> 2.15 or ~> 3.1", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "87748d3c4afe949c7c6eb7150c958c2bcba43fc5b2a02686af30e636b74bccb7"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, + "image": {:hex, :image, "0.55.2", "f21b5341ee05dfe2e0f649c34c6335cbce44be55e3ce3ced404ac008bef6c335", [:mix], [{:bumblebee, "~> 0.3", [hex: :bumblebee, repo: "hexpm", optional: true]}, {:evision, "~> 0.1.33 or ~> 0.2", [hex: :evision, repo: "hexpm", optional: true]}, {:exla, "~> 0.5", [hex: :exla, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:kino, "~> 0.13", [hex: :kino, repo: "hexpm", optional: true]}, {:nx, "~> 0.7", [hex: :nx, repo: "hexpm", optional: true]}, {:nx_image, "~> 0.1", [hex: :nx_image, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.1 or ~> 3.2 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: true]}, {:req, "~> 0.4", [hex: :req, repo: "hexpm", optional: true]}, {:rustler, "> 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:scholar, "~> 0.3", [hex: :scholar, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: false]}, {:vix, "~> 0.23", [hex: :vix, repo: "hexpm", optional: false]}], "hexpm", "aa126e45b514810d1af89eded505ed3e523acefbb005f6220f8fbc1955904607"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, @@ -55,6 +56,7 @@ "pre_commit": {:hex, :pre_commit, "0.3.4", "e2850f80be8090d50ad8019ef2426039307ff5dfbe70c736ad0d4d401facf304", [:mix], [], "hexpm", "16f684ba4f1fed1cba6b19e082b0f8d696e6f1c679285fedf442296617ba5f4e"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "tailwind": {:hex, :tailwind, "0.2.4", "5706ec47182d4e7045901302bf3a333e80f3d1af65c442ba9a9eed152fb26c2e", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "c6e4a82b8727bab593700c998a4d98cf3d8025678bfde059aed71d0000c3e463"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, @@ -62,6 +64,7 @@ "tentacat": {:hex, :tentacat, "2.2.0", "ed2f137c3f64a787cd278ccb1ddbb9e5b696c9c09e3c89d559fffa64ad1494b8", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4ca367af4769774c7dd24a53738f20603012c03715be6c23d8e22c220ee8c07"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "useful": {:hex, :useful, "1.15.0", "3c967dd3013d710538f37ec9ae5d074a4ee19add367172e0dff22c0f72ea30bd", [:mix], [{:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "900180dc839831fa82717ee25ae3d912656095a58fb8a4bed8975495516b8dd0"}, + "vix": {:hex, :vix, "0.33.0", "cd98084529fd8fe3d2336f157db6de03b297fb096508d820068117d58eadb6f1", [:make, :mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:cc_precompiler, "~> 0.1.4 or ~> 0.2", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7.3 or ~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}], "hexpm", "9acde72b27bdfeadeb51f790f7a6cc0d06cf555718c05cf57e43c5cf93d8471b"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.8", "3b97dc94e407e2d1fc666b2fb9acf6be81a1798a2602294aac000260a7c4a47d", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "315b9a1865552212b5f35140ad194e67ce31af45bcee443d4ecb96b5fd3f3782"}, } From d8e17cf9658e3f8a869e71f8e8a00e3a3e9f6b74 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 16 Jan 2025 14:26:00 +0000 Subject: [PATCH 19/58] Img.get_avatar_color(avatar_url) implemented for #221 --- lib/app/image.ex | 26 ++++++++++++++++++++++++-- test/app/image_test.exs | 31 ++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/lib/app/image.ex b/lib/app/image.ex index f4ccf21..bcfcc73 100644 --- a/lib/app/image.ex +++ b/lib/app/image.ex @@ -1,4 +1,4 @@ -defmodule App.Image do +defmodule App.Img do @moduledoc """ Handles extracting color data from images. Uses: https://github.com/elixir-image/image @@ -6,6 +6,7 @@ defmodule App.Image do require Logger :inets.start() :ssl.start() + @hex "0123456789ABCDEF" @doc """ Retrieve raw image data from a URL. @@ -23,6 +24,27 @@ defmodule App.Image do https://hexdocs.pm/image/Image.html#dominant_color/2 """ def extract_color(img_data) do - "blue" + {:ok, [r, g, b]} = + Image.open!(img_data) + |> Image.dominant_color() + + rgb_to_hex(r, g, b) + end + + def get_avatar_color(avatar_url) do + get_raw_image_data(avatar_url) |> extract_color() + end + + # functions borrowed from: + # https://github.com/nelsonic/colors/blob/master/colors.html#L96 + def rgb_to_hex(r, g, b) do + "#{to_hex(r)}#{to_hex(g)}#{to_hex(b)}" + end + + # No error/bounds checking as values come from `Image.dominant_color/1` + def to_hex(n) do + mod = Integer.mod(n, 16) + rounded = trunc(Float.ceil((n - mod) / 16)) + "#{String.at(@hex, rounded)}#{String.at(@hex, mod)}" end end diff --git a/test/app/image_test.exs b/test/app/image_test.exs index a79ac47..b4c8c35 100644 --- a/test/app/image_test.exs +++ b/test/app/image_test.exs @@ -1,17 +1,34 @@ -defmodule App.ImageTest do +defmodule App.ImgTest do use ExUnit.Case - alias App.Image + alias App.Img - test "App.Image.get_raw_image_data/1" do + test "App.Img.get_raw_image_data/1" do url = "https://avatars.githubusercontent.com/u/4185328" - img_data = Image.get_raw_image_data(url) # |> dbg + img_data = Img.get_raw_image_data(url) # |> dbg assert is_binary(img_data) end - test "App.Image.extract_color/1 retrieves color" do + test "App.Img.extract_color/1 retrieves color" do url = "https://avatars.githubusercontent.com/u/4185328" - img_data = Image.get_raw_image_data(url) - Image.extract_color(img_data) + img_data = Img.get_raw_image_data(url) + assert Img.extract_color(img_data) == "F8F8F8" + url2 = "https://avatars.githubusercontent.com/u/194400" + img_data2 = Img.get_raw_image_data(url2) + assert Img.extract_color(img_data2) == "F80818" + end + + test "App.Img.get_avatar_color/1 gets the hex color for avatar" do + avatar_url = "https://avatars.githubusercontent.com/u/4185328" + assert Img.get_avatar_color(avatar_url) == "F8F8F8" + end + + test "to_hex/1 returns the hex value of an integer" do + assert Img.to_hex(42) == "2A" + end + + test "rgb_to_hex/1 returns the hex of an RGB color" do + # https://www.colorhexa.com/2bf0cf + assert Img.rgb_to_hex(43, 240, 207) == "2BF0CF" end end From 4de93bb0a92bf8a5165a3bebb45143386a32aae2 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 16 Jan 2025 18:26:01 +0000 Subject: [PATCH 20/58] add more tests for image dominant color #221 --- lib/app/repository.ex | 1 + test/app/image_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/lib/app/repository.ex b/lib/app/repository.ex index 0418a0a..7292236 100644 --- a/lib/app/repository.ex +++ b/lib/app/repository.ex @@ -47,6 +47,7 @@ defmodule App.Repository do def get_org_repos(org) do App.GitHub.org_repos(org) |> Enum.map(fn repo -> + dbg(repo) create(repo) end) end diff --git a/test/app/image_test.exs b/test/app/image_test.exs index b4c8c35..0adea71 100644 --- a/test/app/image_test.exs +++ b/test/app/image_test.exs @@ -16,11 +16,20 @@ defmodule App.ImgTest do url2 = "https://avatars.githubusercontent.com/u/194400" img_data2 = Img.get_raw_image_data(url2) assert Img.extract_color(img_data2) == "F80818" + + url3 = "https://avatars.githubusercontent.com/u/19310512" + img_data3 = Img.get_raw_image_data(url3) + assert Img.extract_color(img_data3) == "080808" end test "App.Img.get_avatar_color/1 gets the hex color for avatar" do avatar_url = "https://avatars.githubusercontent.com/u/4185328" assert Img.get_avatar_color(avatar_url) == "F8F8F8" + + avatar_url2 = "https://avatars.githubusercontent.com/u/7805691" + assert Img.get_avatar_color(avatar_url2) == "C85878" + # https://github.com/harrygfox + # https://www.color-hex.com/color/c85878 end test "to_hex/1 returns the hex value of an integer" do From 68d4a4ab7b49e1c9d26f8abbd123de29b5d0137a Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 19 Jan 2025 08:43:45 +0000 Subject: [PATCH 21/58] add :hex to user schema for face wall sorting #222 #221 #219 --- lib/app/repository.ex | 22 +++++++++++++++---- lib/app/user.ex | 6 +++-- .../20221005213110_create_users.exs | 1 + test/app/repository_test.exs | 4 +--- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/lib/app/repository.ex b/lib/app/repository.ex index 7292236..d6355eb 100644 --- a/lib/app/repository.ex +++ b/lib/app/repository.ex @@ -25,10 +25,25 @@ defmodule App.Repository do @doc false def changeset(repository, attrs) do + attrs = %{attrs | topics: Enum.join(attrs.topics, ", ")} + repository - |> cast(attrs, [:id, :name, :full_name, :language, :owner_id, :description, - :fork, :forks_count, :watchers_count, :stargazers_count, :topics, - :open_issues_count, :created_at, :pushed_at]) + |> cast(attrs, [ + :created_at, + :id, + :description, + :fork, + :forks_count, + :full_name, + :language, + :name, + :open_issues_count, + :owner_id, + :pushed_at, + :stargazers_count, + :topics, + :watchers_count + ]) |> validate_required([:name, :full_name]) end @@ -47,7 +62,6 @@ defmodule App.Repository do def get_org_repos(org) do App.GitHub.org_repos(org) |> Enum.map(fn repo -> - dbg(repo) create(repo) end) end diff --git a/lib/app/user.ex b/lib/app/user.ex index 646f8cd..ab468cc 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -14,6 +14,7 @@ defmodule App.User do field :email, :string field :followers, :integer field :following, :integer + field :hex, :string field :hireable, :boolean, default: false field :location, :string field :login, :string @@ -27,7 +28,7 @@ defmodule App.User do @doc false def changeset(user, attrs) do user - |> cast(attrs, [:id, :login, :avatar_url, :name, :company, :bio, :blog, :location, :email, :created_at, :hireable, :two_factor_authentication, :public_repos, :followers, :following]) + |> cast(attrs, [:id, :login, :avatar_url, :name, :company, :bio, :blog, :location, :email, :created_at, :hex, :hireable, :two_factor_authentication, :public_repos, :followers, :following]) |> validate_required([:id, :login, :avatar_url]) end @@ -54,6 +55,7 @@ defmodule App.User do else {:ok, user} = map_github_user_fields_to_table(data) + |> Map.put(:hex, App.Img.get_avatar_color(data.avatar_url)) |> create() user @@ -108,7 +110,7 @@ defmodule App.User do def list_users_avatars do from(u in User, select: %{id: u.id}) # |> limit(20) - |> order_by(desc: :inserted_at) + |> order_by(:hex) # |> distinct(true) |> Repo.all() # return a list of urls not a list of maps diff --git a/priv/repo/migrations/20221005213110_create_users.exs b/priv/repo/migrations/20221005213110_create_users.exs index 49f362d..28a8464 100644 --- a/priv/repo/migrations/20221005213110_create_users.exs +++ b/priv/repo/migrations/20221005213110_create_users.exs @@ -12,6 +12,7 @@ defmodule App.Repo.Migrations.CreateUsers do add :followers, :integer add :following, :integer add :hireable, :boolean, default: false + add :hex, :string add :location, :string add :login, :string add :name, :string diff --git a/test/app/repository_test.exs b/test/app/repository_test.exs index fcf2ea1..9758770 100644 --- a/test/app/repository_test.exs +++ b/test/app/repository_test.exs @@ -15,8 +15,7 @@ defmodule App.RepositoryTest do open_issues_count: 98, pushed_at: "2022-08-10T07:41:05Z", stargazers_count: 1604, - topics: Enum.join(["beginner", "beginner-friendly", "how-to", "howto", "learn", - "starter-kit"], ","), + topics: ["beginner", "beginner-friendly", "how-to", "learn"], watchers_count: 1604 } assert {:ok, inserted_repo} = App.Repository.create(repo) @@ -24,7 +23,6 @@ defmodule App.RepositoryTest do end test "App.Repository.get_org_repos/1" do - App.Repository.get_org_repos("ideaq") # |> dbg # assert inserted_repo.name == repo.name end From 5252235facaa7440dc27970c352c3be70a954307 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 20 Jan 2025 12:08:05 +0000 Subject: [PATCH 22/58] create App.Repository.get_repo_id_by_full_name/1 for #223 --- lib/app/github.ex | 12 ++++ lib/app/repository.ex | 14 ++++- lib/app/star.ex | 17 +++++- lib/app_web/live/app_live.html.heex | 61 +++++++++++---------- lib/app_web/templates/layout/root.html.heex | 18 +----- test/app/github_test.exs | 9 ++- test/app/repository_test.exs | 13 +++++ 7 files changed, 95 insertions(+), 49 deletions(-) diff --git a/lib/app/github.ex b/lib/app/github.ex index 285b6c9..69338b0 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -47,4 +47,16 @@ defmodule App.GitHub do Tentacat.Organizations.Members.list(@client, orgname) data end + + @doc """ + `repo_stargazers/2` Returns the list of GitHub users starring a repo. + `owner` - the owner of the repo + `repo` - name of the repo to check stargazers for. + """ + def repo_stargazers(owner, repo) do + Logger.info "Fetching stargazers for #{owner}/#{repo}" + {_status, data, _res} = + Tentacat.Users.Starring.stargazers(@client, owner, repo) + data + end end diff --git a/lib/app/repository.ex b/lib/app/repository.ex index d6355eb..45844f8 100644 --- a/lib/app/repository.ex +++ b/lib/app/repository.ex @@ -1,7 +1,7 @@ defmodule App.Repository do use Ecto.Schema alias App.{Repo} - import Ecto.Changeset + import Ecto.{Changeset, Query} require Logger alias __MODULE__ @@ -56,13 +56,23 @@ defmodule App.Repository do |> Repo.insert() end + @doc """ + `get_repo_id_by_full_name/1` Gets the repository `id` by `full_name`. + e.g: get_repo_id_by_full_name("dwyl/start-here") -> 17338019 + """ + def get_repo_id_by_full_name(full_name) do + from(r in Repository, where: r.full_name == ^full_name, select: r.id) + |> Repo.one() + end + @doc """ Get all repositories for an organization and insert them into DB. """ def get_org_repos(org) do App.GitHub.org_repos(org) |> Enum.map(fn repo -> - create(repo) + {:ok, repo} = create(repo) + repo end) end end diff --git a/lib/app/star.ex b/lib/app/star.ex index 9afa76b..4ac7f4a 100644 --- a/lib/app/star.ex +++ b/lib/app/star.ex @@ -1,7 +1,7 @@ defmodule App.Star do alias App.{Repo} use Ecto.Schema - import Ecto.Changeset + import Ecto.{Changeset, Query} require Logger alias __MODULE__ @@ -28,4 +28,19 @@ defmodule App.Star do |> changeset(attrs) |> Repo.insert() end + + @doc """ + Get all repositories for an organization and insert them into DB. + """ + def get_stargazers_for_repo(owner, repo) do + repo_id = App.Repository.get_repo_id_by_full_name("#{owner}/#{repo}") + App.GitHub.repo_stargazers(owner, repo) + |> Enum.map(fn user -> + dbg(user) + # %{ + + # } + # create(repo) + end) + end end diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index 1a3fc89..4e64c05 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -3,41 +3,44 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round Sync -
+

Newest person:

-
-
-
- - {@data.name} +
+
+
+ + {@data.name} +
+

+ @{@data.login} +

+

+ {@data.name} +

+

+ {truncate_bio(@data.bio)} +

+
    +
  • + Company + + + {@data.company} + + +
    • +
  • + Joined On + {short_date(@data.created_at)} +
    • +
-

- @{@data.login} -

-

- {@data.name} -

-

- {truncate_bio(@data.bio)} -

-
    -
  • - Company - - {@data.company} -
    • -
  • - Joined On - {short_date(@data.created_at)} -
    • -
-
- +
\ No newline at end of file diff --git a/lib/app_web/templates/layout/root.html.heex b/lib/app_web/templates/layout/root.html.heex index c0d5656..9d2a250 100644 --- a/lib/app_web/templates/layout/root.html.heex +++ b/lib/app_web/templates/layout/root.html.heex @@ -12,10 +12,10 @@ - +
# **`TODO`**: re-generate the "wall of faces" using latest data `#HelpWanted` ![face wall](https://user-images.githubusercontent.com/194400/28011265-a95f52d4-6559-11e7-823e-6133d947921a.jpg) - # *Why*? -We needed an **easy, fast & reliable _system_** +We needed an **easy, fast & reliable _system_** to **_visualize_ `who`** is joining the **`@dwyl` community**
and **track growth** over time. πŸ“ˆ -The [**start-here** > ***who***](https://github.com/dwyl/start-here/tree/8bbd28d2ab0c3b5a2a266a1e41fd160fc6ee3038#who) +The [**start-here** > ***who***](https://github.com/dwyl/start-here/tree/8bbd28d2ab0c3b5a2a266a1e41fd160fc6ee3038#who) section ~~is~~ _was_ *woefully* out of date because we had to update it _manually_. ⏳
-(_this was -[noted](https://github.com/dwyl/start-here/issues/9) -a while back... -but sadly was not made +(_this was +[noted](https://github.com/dwyl/start-here/issues/9) +a `while` back... +but sadly was not made a priority at the time..._) -This mini-app/project is designed +This mini-app/project is designed to scratch our own itch and save us [time](https://github.com/dwyl/start-here/issues/255). # *What*? -There are **_two_ ways** -of discovering -the list of people -contributing to the -**dwyl mission**: +There are **_two_ ways** +of discovering +the list of people +contributing to the +**dwyl mission**; + ## 1. _Manually_ check *dwyl* Org *People Page* on GitHub Visit @@ -59,24 +57,24 @@ who are _members_ of the Org. *Simple. effective. incomplete*. This list only scratches the surface! -## 2. List all contributors to dwyl repos on GitHub +## 2. List all contributors to dwyl repos on `GitHub` Read the Commit History for all the dwyl repos on GitHub and extract the names of people ...
-As you can imagine, -this second option +As you can imagine, +this second option is _painful_ to do _manually_ ... ⏳
-So we _had_ to create a mini-App -to do it for us +So we _had_ to create a mini-App +to do it for us via the **`GitHub` API**! πŸ’‘ # *How*? -We built this mini-App -using the +We built this mini-App +using the [**`PETAL`** Stack](https://github.com/dwyl/technology-stack/#the-petal-stack) because we feel
-it's the _fastest_ +it's the _fastest_ and most _effective_ way to ship a web app. @@ -84,24 +82,24 @@ to ship a web app. If you want to **understand _every_ step** of the process of **_building_** the **mini-app**, -read: +read: [**`BUILDIT.md`**](https://github.com/dwyl/who/blob/main/BUILDIT.md) ## Run the `Who` App on your `localhost` ⬇️ -> **Note**: You will need to have +> **Note**: You will need to have **`Elixir`** and **`Postgres` installed**,
-see: +see: [learn-elixir#installation](https://github.com/dwyl/learn-elixir#installation) -and +and [learn-postgresql#installation](https://github.com/dwyl/learn-postgresql#installation) > respectively.
> **Tip**: check the prerequisites in: > [**/phoenix-chat-example**](https://github.com/dwyl/phoenix-chat-example#0-pre-requisites-before-you-start) -On your `localhost`, -run the following commands +On your `localhost`, +run the following commands in your terminal: ```sh @@ -129,12 +127,23 @@ required to run the App. ### Get your `GitHub` Personal Access Token To access the **`GitHub` API**, -you will need to generate a +you will need to generate a **Personal Access Token**: -[github.com/settings/tokens](https://github.com/settings/tokens) +[github.com/settings/tokens](https://github.com/settings/tokens/new) Click on the **`Generate new token`** button. -Name it something memorable. +Name it something memorable so you know what the token is for: + +github-token-name + +and make sure the token will have both `repo` + +repo-access + +and `user` access: + +user-access + Once you've created the token, copy it to your clipboard for the next step. @@ -162,8 +171,7 @@ mix s Open the App in your web browser [**`localhost:4000`**](http://localhost:4000/) -and start your tour! - +and start your tour!
@@ -175,11 +183,11 @@ to feature requests are always welcome! πŸ™Œ Please start by:
-a. **Star** the repo on GitHub +a. **Star** the repo on GitHub so you have a "bookmark" you can return to. ⭐
-b. **Fork** the repo +b. **Fork** the repo so you have a copy you can "hack" on. 🍴
-c. **Clone** the repo to your `localhost` +c. **Clone** the repo to your `localhost` and run it! πŸ‘©β€πŸ’»
@@ -190,15 +198,14 @@ please see: ### More Features? πŸ”” If you have feature ideas, that's great! πŸŽ‰
-Please _share_: +Please _share_: [**who/issues**](https://github.com/dwyl/who/issues) πŸ™ - # Features (Todo) -+ List Repos in the Org: ++ List Repos in the Org: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories -+ List of people that Star a given repo: ++ List of people that Star a given repo. -+ \ No newline at end of file ++ List of people who have _contributed_ to repo. \ No newline at end of file From 3e908ed5f90ac9468a884ca0591e91fb57aa7e4c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 27 Jan 2025 11:58:53 +0000 Subject: [PATCH 26/58] add GitHub Api instructions to BUILDIT.md #207 --- BUILDIT.md | 239 ++++++++++------------------------- README.md | 3 +- lib/app_web/live/app_live.ex | 2 +- 3 files changed, 67 insertions(+), 177 deletions(-) diff --git a/BUILDIT.md b/BUILDIT.md index b552c98..557a8d1 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -175,15 +175,15 @@ That tells us everything is working as expected. πŸš€ If you prefer to see **test coverage** - we certainly do - then you will need to add a few lines to the [`mix.exs`](https://github.com/dwyl/who/blob/main/mix.exs) -file and create a +file and create a [`coveralls.json`](https://github.com/dwyl/who/blob/main/coveralls.json) file to exclude `Phoenix` files from `excoveralls` checking. -Add alias (shortcuts) in `mix.exs` `defp aliases do` list. +Add alias (shortcuts) in `mix.exs` `defp aliases do` list. -e.g: `mix c` runs `mix coveralls.html` +e.g: `mix c` runs `mix coveralls.html` see: [**`commits/d6ab5ef`**](https://github.com/dwyl/app-mvp/pull/90/commits/d6ab5ef7c2be5dcad7d060e782393ae29c94a526) ... -This is just standard `Phoenix` project setup for us, +This is just standard `Phoenix` project setup for us, so we don't duplicate any of the steps here.
For more detail, please see: [Automated Testing](https://github.com/dwyl/phoenix-chat-example#testing-our-app-automated-testing) @@ -202,7 +202,6 @@ You should see output similar to the following: Who tests passing coverage 100% - ## 1.3 Setup `Tailwind` As we're using **`Tailwind CSS`** @@ -215,10 +214,10 @@ please refer to: Should only take **`~1 minute`**. By the end of this step you should have **`Tailwind`** working. -When you visit -[`localhost:4000`](http://localhost:4000) -in your browser, -you should see: +When you visit +[`localhost:4000`](http://localhost:4000) +in your browser, +you should see: ![hello world tailwind phoenix](https://user-images.githubusercontent.com/194400/174838767-20bf201e-3179-4ff9-8d5d-751295d1d069.png) @@ -252,7 +251,7 @@ end Next, create the **`lib/app_web/live/app_live.html.heex`** -file +file and add the following line of `HTML`: ```html @@ -263,9 +262,9 @@ and add the following line of `HTML`: ``` Finally, to make the **root layout** simpler, -open the +open the `lib/app_web/templates/layout/root.html.heex` -file and +file and update the contents of the `` to: ```html @@ -283,7 +282,7 @@ update the contents of the `` to: Now that you've created the necessary files, open the router -`lib/app_web/router.ex` +`lib/app_web/router.ex` replace the default route `PageController` controller: ```elixir @@ -337,10 +336,10 @@ Finished in 0.1 seconds (0.06s async, 0.1s sync) 3 tests, 1 failure ``` -Create a new directory: +Create a new directory: `test/app_web/live` -Then create the file: +Then create the file: `test/app_web/live/app_live_test.exs` With the following content: @@ -356,7 +355,7 @@ defmodule AppWeb.AppLiveTest do end ``` -Save the file +Save the file and re-run the tests: `mix test` You should see output similar to the following: @@ -374,7 +373,7 @@ Randomized with seed 796477 ## 1.7 Delete Page-related Files -Since we won't be using the `page` in our App, +Since we won't be using the `page` in our App, we can delete the default files created by `Phoenix`: ```sh @@ -391,7 +390,7 @@ and ready to start _building_! ## Run Tests with Coverage -``` +```sh mix c ``` @@ -418,11 +417,16 @@ COV FILE LINES RELEVANT MISSED This app stores data in **five** schemas: -1. `users` - https://docs.github.com/en/rest/users/users - the GitHub [**`users`**](https://dwyl.github.io/book/auth/07-notes-on-naming.html) that _use_ the platform. +1. `users` - https://docs.github.com/en/rest/users/users - the GitHub + [**`users`**](https://dwyl.github.io/book/auth/07-notes-on-naming.html) + that _use_ the platform. 2. `orgs` - [https://docs.github.com/en/rest/orgs/orgs](https://docs.github.com/en/rest/orgs/orgs?#get-an-organization) - organizations which can have `users` as members and `repositories`. -3. `repositories` - https://docs.github.com/en/rest/repos/repos - the repositories of code on GitHub. -4. `stars` - [https://docs.github.com/en/rest/activity/starring](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) - the `stars` (on `repositories`) associated with each `user`. -5. `follows` - https://docs.github.com/en/rest/users/followers - List the `people` a `user` follows. +3. `repositories` - https://docs.github.com/en/rest/repos/repos - + the repositories of code on GitHub. +4. `stars` - [https://docs.github.com/en/rest/activity/starring](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) - + the `stars` (on `repositories`) associated with each `user`. +5. `follows` - https://docs.github.com/en/rest/users/followers - + List the `people` a `user` follows. For each of these schemas we are storing a _subset_ of the data; @@ -454,7 +458,6 @@ we have the following database [Entity Relationship Diagram](https://en.wikipedia.org/wiki/Entity%E2%80%93relationship_model) (ERD): - ![erd](https://user-images.githubusercontent.com/194400/194425189-e44d6161-c8df-4a0d-9d86-bc1045785c95.png) We created **5 database tables**; @@ -463,7 +466,6 @@ At present the two tables are unrelated but eventually `repository.owner_id` will refer to `user.id` and we will be creating other schemas below. -
## 2.1 Run Tests! @@ -476,7 +478,7 @@ If we run the tests with coverage: mix c ``` -We note that the test coverage +We note that the test coverage has dropped considerably: ```sh @@ -681,188 +683,71 @@ COV FILE LINES RELEVANT MISSED We have our starting point for the project, let's write some code! +
+# 3. Setup `GitHub` API - -




- -# 3. Setup `GitHub` API - -We're going to make _many_ requests +We're going to make _many_ requests to the **`GitHub` REST API`**. So we need an effective way of doing that. - - - Create the directory `test/app` and file: -`test/app/item_test.exs` +`test/app/github_test.exs` with the following code: ```elixir -defmodule App.ItemTest do - use App.DataCase - alias App.{Item, Timer} - - describe "items" do - @valid_attrs %{text: "some text", person_id: 1, status: 2} - @update_attrs %{text: "some updated text"} - @invalid_attrs %{text: nil} - - test "get_item!/1 returns the item with given id" do - {:ok, item} = Item.create_item(@valid_attrs) - assert Item.get_item!(item.id).text == item.text - end - - test "create_item/1 with valid data creates a item" do - assert {:ok, %Item{} = item} = Item.create_item(@valid_attrs) - - assert item.text == "some text" +defmodule App.GitHubTest do + use ExUnit.Case + alias App.GitHub - inserted_item = List.first(Item.list_items()) - assert inserted_item.text == @valid_attrs.text - end - - test "create_item/1 with invalid data returns error changeset" do - assert {:error, %Ecto.Changeset{}} = - Item.create_item(@invalid_attrs) - end - - test "list_items/0 returns a list of items stored in the DB" do - {:ok, _item1} = Item.create_item(@valid_attrs) - {:ok, _item2} = Item.create_item(@valid_attrs) - - assert Enum.count(Item.list_items()) == 2 - end - - test "update_item/2 with valid data updates the item" do - {:ok, item} = Item.create_item(@valid_attrs) - - assert {:ok, %Item{} = item} = Item.update_item(item, @update_attrs) - assert item.text == "some updated text" - end + test "App.GitHub.user/1" do + username = "iteles" + user = GitHub.user(username) + assert user.public_repos > 30 end end + ``` -The first five tests are basic -[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete). +The first test is very basic; +just fetches a `user` from the `GitHub` API +and confirms they have more than 30 `public_repos`. If you run these tests: + ```sh -mix test test/app/item_test.exs +mix test test/app/github_test.exs ``` You will see all the testes _fail_. This is expected as the code is not there yet! - - ## 3.1 Make the `user` Tests Pass -Open the -`lib/app/item.ex` -file and replace the contents +Open the +`lib/app/github.ex` +file and replace the contents with the following code: - ```elixir -defmodule App.Item do - use Ecto.Schema - import Ecto.Changeset - import Ecto.Query - alias App.Repo - alias __MODULE__ - - schema "items" do - field :person_id, :integer - field :status, :integer - field :text, :string - - timestamps() - end - - @doc false - def changeset(item, attrs) do - item - |> cast(attrs, [:person_id, :status, :text]) - |> validate_required([:text]) - end - - @doc """ - Creates a item. - - ## Examples - - iex> create_item(%{text: "Learn LiveView"}) - {:ok, %Item{}} - - iex> create_item(%{text: nil}) - {:error, %Ecto.Changeset{}} - - """ - def create_item(attrs) do - %Item{} - |> changeset(attrs) - |> Repo.insert() - end - - @doc """ - Gets a single item. - - Raises `Ecto.NoResultsError` if the Item does not exist. - - ## Examples - - iex> get_item!(123) - %Item{} - - iex> get_item!(456) - ** (Ecto.NoResultsError) - +defmodule App.GitHub do + @moduledoc """ + Handles all interactions with the GitHub REST API + via: github.com/edgurgel/tentacat Elixir GitHub Lib. """ - def get_item!(id), do: Repo.get!(Item, id) - - @doc """ - Returns the list of items where the status is different to "deleted" - - ## Examples + require Logger - iex> list_items() - [%Item{}, ...] - - """ - def list_items do - Item - |> order_by(desc: :inserted_at) - |> where([i], is_nil(i.status) or i.status != 6) - |> Repo.all() - end + @access_token Application.compile_env(:tentacat, :access_token) + @client Tentacat.Client.new(%{access_token: @access_token}) @doc """ - Updates an `item`. - - ## Examples - - iex> update_item(item, %{field: new_value}) - {:ok, %Item{}} - - iex> update_item(item, %{field: bad_value}) - {:error, %Ecto.Changeset{}} - + `user/1` Returns the GitHub user profile data. """ - def update_item(%Item{} = item, attrs) do - item - |> Item.changeset(attrs) - |> Repo.update() - end - - # soft delete an item: - def delete_item(id) do - get_item!(id) - |> Item.changeset(%{status: 6}) - |> Repo.update() + def user(username) do + Logger.info "Fetching user #{username}" + {_status, data, _res} = Tentacat.Users.find(@client, username) + data end end ``` @@ -870,6 +755,12 @@ end Once you have saved the file, re-run the tests. They should now pass. +> **Note**: using the `GitHub` API assumes you already +> have a personal access token defined +> as an environment variable `GH_PERSONAL_ACCESS_TOKEN` +> if you don't please see: +> [dwyl/who#get-your-github-personal-access-token](https://github.com/dwyl/who?tab=readme-ov-file#get-your-github-personal-access-token) + # X. Add Authentication This section borrows heavily from: diff --git a/README.md b/README.md index bb48571..8e05205 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,8 @@ Create an `.env` file by copying the sample: ```sh cp .env_sample .env ``` -Paste the value of your -loads the +This file will load the [environment variables](https://github.com/dwyl/learn-environment-variables) required to run the App. diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index bb648ba..add18a4 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -47,7 +47,7 @@ defmodule AppWeb.AppLive do App.Repository.get_org_repos(org) # get all stargazers for a given repo |> Enum.map(fn repo -> - dbg(repo) + # dbg(repo) # App.Star.get_stargazers_for_repo(owner, repo) repo end) From 2223203ffbc9062942e95c066182a905f54e10c7 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 27 Jan 2025 20:47:46 +0000 Subject: [PATCH 27/58] add code for API request logging #226 --- BUILDIT.md | 138 ++++++++++++++++++ README.md | 12 +- lib/app/github.ex | 17 ++- lib/app/reqlog.ex | 37 +++++ .../20250127180724_create_reqlogs.exs | 12 ++ test/app/github_test.exs | 1 + test/app/reqlog_test.exs | 24 +++ 7 files changed, 230 insertions(+), 11 deletions(-) create mode 100644 lib/app/reqlog.ex create mode 100644 priv/repo/migrations/20250127180724_create_reqlogs.exs create mode 100644 test/app/reqlog_test.exs diff --git a/BUILDIT.md b/BUILDIT.md index 557a8d1..a8f34c4 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -64,6 +64,7 @@ where (_hopefully_) it will all be clear. - [2.2.3 Re-run the Tests](#223-re-run-the-tests) - [3. Setup `GitHub` API](#3-setup-github-api) - [3.1 Make the `user` Tests Pass](#31-make-the-user-tests-pass) +- [4. Log All `GitHub` API Request](#4-log-all-github-api-request) - [X. Add Authentication](#x-add-authentication) - [X.1 Add `auth_plug` to `deps`](#x1-add-auth_plug-to-deps) - [X.2 Get your `AUTH_API_KEY`](#x2-get-your-auth_api_key) @@ -761,6 +762,143 @@ They should now pass. > if you don't please see: > [dwyl/who#get-your-github-personal-access-token](https://github.com/dwyl/who?tab=readme-ov-file#get-your-github-personal-access-token) +# 4. Log All `GitHub` API Request + +As noted in +[who#226](https://github.com/dwyl/who/issues/226) +the `GitHub` API is rate-limited to +`5,000 requests per hour`. +We would immediately exhaust this limit in a minute +and be **blocked** with a +[`429 Error`](https://www.rfc-editor.org/rfc/rfc6585#section-4) 🚫 +if we make all the requests we need in one go. +So instead we need to _log_ all the requests +so that we know not to exceed the `5k/h` limit. + +Create the `Request Log` (`Reqlog`) schema with the following command: + +```sh +mix phx.gen.schema Reqlog reqlogs req:string param:string +``` + +The output is: + +```sh +* creating lib/app/reqlog.ex +* creating priv/repo/migrations/20250127180724_create_reqlogs.exs +``` + +But for whatever reason it doesn't automatically the test file. +Create it manually with: + +```sh +vi test/app/reqlog_test.exs +``` + +And _paste_ the following code: + +```elixir +defmodule App.ReqlogTest do + use App.DataCase + + test "App.Reqlog.create/1" do + owner = "dwyl" + reponame = "mvp" + record = %{ + created_at: "2014-03-02T13:20:04Z", + req: "repository", + param: "#{owner}/#{reponame}" + } + assert {:ok, inserted} = App.Reqlog.create(record) + assert inserted.req == record.req + end + + test "App.Reqlog.log/2" do + owner = "dwyl" + reponame = "mvp" + + assert {:ok, inserted} = App.Reqlog.log("repo", "#{owner}/#{reponame}") + assert inserted.req == "repo" + assert inserted.param == "#{owner}/#{reponame}" + end +end +``` + +Running the test will fail: + +```sh +mix test test/app/reqlog_test.exs +``` + +Open the `lib/app/reqlog.ex` file +and add the following code: + +```elixir + @doc """ + Creates a `reqlog` (request log) record. + """ + def create(attrs) do + %Reqlog{} + |> changeset(attrs) + |> Repo.insert() + end + + def log(req, param) do + Logger.info "Fetching #{req} #{param}" + create(%{req: req, param: param}) + end +``` + +At the top of the file under the `defmodule` add these two lines: + +```sh + alias App.{Repo} + alias __MODULE__ + require Logger +``` + +One allows us to use `Repo.insert/1` +`alias __MODULE__` defines an alias for our Elixir module. +`require Logger` loads the [`Logger`](https://hexdocs.pm/logger). + +Ref: +https://alphahydrae.com/2021/03/how-to-make-an-elixir-module-alias-itself/ + +With that code in-place, we can _use_ it in our `GitHub` file. + +Remember to add the line: + +```elixir + use App.DataCase +``` + +to the top of the file `test/app/github_test.exs` +to ensure that the `GitHub` API requests get logged during testing. + +In the `lib/app/github.ex` file, +add the following line to the top: + +```elixir +import App.Reqlog, only: [log: 2] +``` + +Then replace the line: + +```elixir +Logger.info "Fetching repository #{owner}/#{reponame}" +``` + +with: + +```elixir +log("repository", "#{owner}/#{reponame}") +``` + +Replace any other instances of `Logger.info` with `log/2`. + + + + # X. Add Authentication This section borrows heavily from: diff --git a/README.md b/README.md index 8e05205..234ff71 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![who-banner](https://user-images.githubusercontent.com/194400/194710724-0e2de0b1-0b2a-4ee8-83a0-eb07cce74810.png) -The **quick _answer_** +The **quick _answer_** to the question: **_Who_ is in the `@dwyl` community?** @@ -18,6 +18,7 @@ to the question: # **`TODO`**: re-generate the "wall of faces" using latest data `#HelpWanted` + ![face wall](https://user-images.githubusercontent.com/194400/28011265-a95f52d4-6559-11e7-823e-6133d947921a.jpg) # *Why*? @@ -52,9 +53,9 @@ contributing to the Visit [github.com/orgs/dwyl/people](https://github.com/orgs/dwyl/people) -you can see a list of people -who are _members_ of the Org. -*Simple. effective. incomplete*. +you can see a list of people +who are _members_ of the Org. +*Simple. effective. incomplete*. This list only scratches the surface! ## 2. List all contributors to dwyl repos on `GitHub` @@ -106,7 +107,8 @@ in your terminal: git clone git@github.com:dwyl/who.git && cd who mix setup ``` -That will download the **`code`**, + +That will download the **`code`**, install dependencies and create the necessary database + tables. diff --git a/lib/app/github.ex b/lib/app/github.ex index 69338b0..1b77b6c 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -3,7 +3,7 @@ defmodule App.GitHub do Handles all interactions with the GitHub REST API via: github.com/edgurgel/tentacat Elixir GitHub Lib. """ - require Logger + import App.Reqlog, only: [log: 2] @access_token Application.compile_env(:tentacat, :access_token) @client Tentacat.Client.new(%{access_token: @access_token}) @@ -13,7 +13,8 @@ defmodule App.GitHub do Returns the GitHub repository data. """ def repository(owner, reponame) do - Logger.info "Fetching repository #{owner}/#{reponame}" + log("repository", "#{owner}/#{reponame}") + # Logger.info "Fetching repository #{owner}/#{reponame}" {_status, data, _res} = Tentacat.Repositories.repo_get(@client, owner, reponame) data @@ -23,7 +24,8 @@ defmodule App.GitHub do Returns the list of GitHub repositories for an Org. """ def org_repos(owner) do - Logger.info "Fetching list of repositories for #{owner}" + log("org_repos", owner) + # Logger.info "Fetching list of repositories for #{owner}" {_status, data, _res} = Tentacat.Repositories.list_orgs(@client, owner) data @@ -33,7 +35,8 @@ defmodule App.GitHub do `user/1` Returns the GitHub user profile data. """ def user(username) do - Logger.info "Fetching user #{username}" + # Logger.info "Fetching user #{username}" + log("user", username) {_status, data, _res} = Tentacat.Users.find(@client, username) data end @@ -42,7 +45,8 @@ defmodule App.GitHub do `org_user_list/1` Returns the list of GitHub users for an org. """ def org_user_list(orgname) do - Logger.info "Fetching org user list for #{orgname}" + # Logger.info "Fetching org user list for #{orgname}" + log("org_user_list", orgname) {_status, data, _res} = Tentacat.Organizations.Members.list(@client, orgname) data @@ -54,7 +58,8 @@ defmodule App.GitHub do `repo` - name of the repo to check stargazers for. """ def repo_stargazers(owner, repo) do - Logger.info "Fetching stargazers for #{owner}/#{repo}" + log("repo_stargazers", "#{owner}/#{repo}") + # Logger.info "Fetching stargazers for #{owner}/#{repo}" {_status, data, _res} = Tentacat.Users.Starring.stargazers(@client, owner, repo) data diff --git a/lib/app/reqlog.ex b/lib/app/reqlog.ex new file mode 100644 index 0000000..4b584d2 --- /dev/null +++ b/lib/app/reqlog.ex @@ -0,0 +1,37 @@ +defmodule App.Reqlog do + use Ecto.Schema + import Ecto.Changeset + alias App.{Repo} + alias __MODULE__ + require Logger + + schema "reqlogs" do + field :req, :string + field :param, :string + + timestamps() + end + + @doc false + def changeset(reqlog, attrs) do + reqlog + |> cast(attrs, [:req, :param]) + |> validate_required([:req, :param]) + end + + @doc """ + Creates a `reqlog` (request log) record. + """ + def create(attrs) do + %Reqlog{} + |> changeset(attrs) + |> Repo.insert() + end + + # clean interface function that can be called from App.GitHub functions + # e.g: log("repository", "#{owner}/#{reponame}") + def log(req, param) do + Logger.info "Fetching #{req} #{param}" + create(%{req: req, param: param}) + end +end diff --git a/priv/repo/migrations/20250127180724_create_reqlogs.exs b/priv/repo/migrations/20250127180724_create_reqlogs.exs new file mode 100644 index 0000000..b3b7c93 --- /dev/null +++ b/priv/repo/migrations/20250127180724_create_reqlogs.exs @@ -0,0 +1,12 @@ +defmodule App.Repo.Migrations.CreateReqlogs do + use Ecto.Migration + + def change do + create table(:reqlogs) do + add :req, :string + add :param, :string + + timestamps() + end + end +end diff --git a/test/app/github_test.exs b/test/app/github_test.exs index 9b8821f..9736830 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -1,5 +1,6 @@ defmodule App.GitHubTest do use ExUnit.Case + use App.DataCase alias App.GitHub test "App.GitHub.repository/1" do diff --git a/test/app/reqlog_test.exs b/test/app/reqlog_test.exs new file mode 100644 index 0000000..f382372 --- /dev/null +++ b/test/app/reqlog_test.exs @@ -0,0 +1,24 @@ +defmodule App.ReqlogTest do + use App.DataCase + + test "App.Reqlog.create/1" do + owner = "dwyl" + reponame = "mvp" + record = %{ + created_at: "2014-03-02T13:20:04Z", + req: "repository", + param: "#{owner}/#{reponame}" + } + assert {:ok, inserted} = App.Reqlog.create(record) + assert inserted.req == record.req + end + + test "App.Reqlog.log/2" do + owner = "dwyl" + reponame = "mvp" + + assert {:ok, inserted} = App.Reqlog.log("repo", "#{owner}/#{reponame}") + assert inserted.req == "repo" + assert inserted.param == "#{owner}/#{reponame}" + end +end From d700665fefa41e34ca6c0a25e259e84a1ff5cc2f Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 28 Jan 2025 12:11:41 +0000 Subject: [PATCH 28/58] create Reqlog.req_count_last_hour/0 for #226 --- BUILDIT.md | 4 ++++ lib/app/api_manager.ex | 32 ++++++++++++++++++++++++++++++++ lib/app/reqlog.ex | 11 ++++++++++- lib/app/star.ex | 3 +++ test/app/reqlog_test.exs | 6 ++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 lib/app/api_manager.ex diff --git a/BUILDIT.md b/BUILDIT.md index a8f34c4..f57ab00 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -65,6 +65,7 @@ where (_hopefully_) it will all be clear. - [3. Setup `GitHub` API](#3-setup-github-api) - [3.1 Make the `user` Tests Pass](#31-make-the-user-tests-pass) - [4. Log All `GitHub` API Request](#4-log-all-github-api-request) + - [4.1 Limit API Requests](#41-limit-api-requests) - [X. Add Authentication](#x-add-authentication) - [X.1 Add `auth_plug` to `deps`](#x1-add-auth_plug-to-deps) - [X.2 Get your `AUTH_API_KEY`](#x2-get-your-auth_api_key) @@ -896,6 +897,9 @@ log("repository", "#{owner}/#{reponame}") Replace any other instances of `Logger.info` with `log/2`. +## 4.1 Limit API Requests + + diff --git a/lib/app/api_manager.ex b/lib/app/api_manager.ex new file mode 100644 index 0000000..cb0af2f --- /dev/null +++ b/lib/app/api_manager.ex @@ -0,0 +1,32 @@ +defmodule App.ApiManager do + @moduledoc """ + This function manages requests to the `GitHub` API + To avoid hitting their `5k/h` limit and being blocked. + Inspired by José Valim's: https://stackoverflow.com/a/32097971/1148249 + """ + use GenServer + + def start_link(_opts) do + GenServer.start_link(__MODULE__, %{}) + end + + def init(state) do + schedule_work() # Schedule work to be performed at some point + {:ok, state} + end + + def handle_info(:work, state) do + # Do the work you desire here + schedule_work() # Reschedule once more + {:noreply, state} + end + + defp schedule_work() do + Process.send_after(self(), :work, 2 * 60 * 60 * 1000) # In 2 hours + end + + def get_users() do + # Check how many requests have been made in the last hour: + + end +end diff --git a/lib/app/reqlog.ex b/lib/app/reqlog.ex index 4b584d2..b195ef1 100644 --- a/lib/app/reqlog.ex +++ b/lib/app/reqlog.ex @@ -1,6 +1,6 @@ defmodule App.Reqlog do use Ecto.Schema - import Ecto.Changeset + import Ecto.{Changeset, Query} alias App.{Repo} alias __MODULE__ require Logger @@ -34,4 +34,13 @@ defmodule App.Reqlog do Logger.info "Fetching #{req} #{param}" create(%{req: req, param: param}) end + + def req_count_last_hour() do + # Using `DateTime.add/4` with a negative number to subtract. ;-) + # via: https://elixirforum.com/t/create-time-with-one-hour-plus/3666/5 + one_hour_ago = DateTime.utc_now(:second) |> DateTime.add(-3600) + + Repo.one(from r in Reqlog, select: count("*"), + where: r.inserted_at > ^one_hour_ago) + end end diff --git a/lib/app/star.ex b/lib/app/star.ex index e2ef0d5..3bdfb52 100644 --- a/lib/app/star.ex +++ b/lib/app/star.ex @@ -37,6 +37,9 @@ defmodule App.Star do repo_id = App.Repository.get_repo_id_by_full_name("#{owner}/#{repo}") App.GitHub.repo_stargazers(owner, repo) |> Enum.map(fn user -> + # We have multiple repos over 1k stars + # Therefore issuing all these requests at once + # would instantly hit the 5k/h GitHub API Request Limit App.User.get_user_from_api(user) {:ok, star} = create(%{ user_id: user.id, repo_id: repo_id }) diff --git a/test/app/reqlog_test.exs b/test/app/reqlog_test.exs index f382372..ac332e9 100644 --- a/test/app/reqlog_test.exs +++ b/test/app/reqlog_test.exs @@ -21,4 +21,10 @@ defmodule App.ReqlogTest do assert inserted.req == "repo" assert inserted.param == "#{owner}/#{reponame}" end + + test "App.Reqlog.req_count_last_hour/0" do + assert {:ok, _} = App.Reqlog.log("repo", "dwyl/any") + count = App.Reqlog.req_count_last_hour() + assert count > 0 + end end From 3c826297e04523f0d4243e67febf7437bd78f20c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 28 Jan 2025 18:22:50 +0000 Subject: [PATCH 29/58] expose request count in LiveView to visualise #226 --- lib/app/api_manager.ex | 6 +++++- lib/app/reqlog.ex | 1 - lib/app_web/live/app_live.ex | 5 +++-- lib/app_web/live/app_live.html.heex | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/app/api_manager.ex b/lib/app/api_manager.ex index cb0af2f..275e665 100644 --- a/lib/app/api_manager.ex +++ b/lib/app/api_manager.ex @@ -22,11 +22,15 @@ defmodule App.ApiManager do end defp schedule_work() do - Process.send_after(self(), :work, 2 * 60 * 60 * 1000) # In 2 hours + Process.send_after(self(), :work, 60 * 1000) # check again in 1 min end def get_users() do # Check how many requests have been made in the last hour: + count = App.Reqlog.req_count_last_hour() + if count < 4920 do + # Get the top 80 users that need to be queried + end end end diff --git a/lib/app/reqlog.ex b/lib/app/reqlog.ex index b195ef1..81796be 100644 --- a/lib/app/reqlog.ex +++ b/lib/app/reqlog.ex @@ -39,7 +39,6 @@ defmodule App.Reqlog do # Using `DateTime.add/4` with a negative number to subtract. ;-) # via: https://elixirforum.com/t/create-time-with-one-hour-plus/3666/5 one_hour_ago = DateTime.utc_now(:second) |> DateTime.add(-3600) - Repo.one(from r in Reqlog, select: count("*"), where: r.inserted_at > ^one_hour_ago) end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index add18a4..04e57a9 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -15,8 +15,9 @@ defmodule AppWeb.AppLive do created_at: "2010-02-02T08:44:49Z", company: "ideaq"} # NEXT: prepend avatars to list ... - - {:ok, assign(socket, %{data: p, ids: App.User.list_users_avatars()})} + {:ok, assign(socket, %{data: p, + ids: App.User.list_users_avatars(), + count: App.Reqlog.req_count_last_hour()})} end def handle_event("sync", value, socket) do diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index 4e64c05..4775d3b 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -2,6 +2,7 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 rounded"> Sync +

{@count}

Newest person:

From 9a86ea3376b1d598bca8e48559dc3de1a3165367 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 29 Jan 2025 07:53:51 +0000 Subject: [PATCH 30/58] attempt to use Elixir 1.18.2 and OTP 27.2 in ci for #229 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91d7269..2645f4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,8 @@ jobs: - name: Set up Elixir uses: erlef/setup-beam@v1 # https://github.com/erlef/setup-beam with: - elixir-version: '1.14.3' # Define the elixir version [required] - otp-version: '25.0' # Define the OTP version [required] + elixir-version: '1.18.2' # Define the elixir version [required] + otp-version: '27.2' # Define the OTP version [required] - name: Restore dependencies cache uses: actions/cache@v2 with: From 0474e815d04d51571fc972db44f4d7d757fd9510 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 29 Jan 2025 08:03:32 +0000 Subject: [PATCH 31/58] add docs for req_count_last_hour/0 #226 --- lib/app/reqlog.ex | 8 +++++++- lib/app_web/live/app_live.ex | 3 ++- mix.exs | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/app/reqlog.ex b/lib/app/reqlog.ex index 81796be..c40da79 100644 --- a/lib/app/reqlog.ex +++ b/lib/app/reqlog.ex @@ -29,12 +29,18 @@ defmodule App.Reqlog do end # clean interface function that can be called from App.GitHub functions - # e.g: log("repository", "#{owner}/#{reponame}") + # e.g: log("repository", "#{owner}/#{repo}") def log(req, param) do Logger.info "Fetching #{req} #{param}" create(%{req: req, param: param}) end + @doc """ + `req_count_last_hour/0` returns the count (integer) of how many API Requests + were made in the last hour to help us stay under the `5k/h` limit. + Sample SQL if you need to test this independently: + SELECT COUNT(*) FROM reqlogs WHERE inserted_at > '2025-01-27 11:02:50' + """ def req_count_last_hour() do # Using `DateTime.add/4` with a negative number to subtract. ;-) # via: https://elixirforum.com/t/create-time-with-one-hour-plus/3666/5 diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 04e57a9..80fb131 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -62,7 +62,8 @@ defmodule AppWeb.AppLive do # IO.inspect("- - - Enum.map u.login: #{index}: #{u.login}") data = App.User.get_user_from_api(u) |> AuthPlug.Helpers.strip_struct_metadata() - new_state = assign(socket, %{data: data}) + sock = assign(socket, %{data: data}) + new_state = assign(sock, %{count: App.Reqlog.req_count_last_hour()}) Task.start(fn -> :timer.sleep(300 + 100 * index) AppWeb.Endpoint.broadcast(@topic, "update", new_state.assigns) diff --git a/mix.exs b/mix.exs index 0ac034d..3364239 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule App.MixProject do [ app: :app, version: "1.7.0", - elixir: "~> 1.14", + elixir: "~> 1.17", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), start_permanent: Mix.env() == :prod, From 321b81de7592695aaeefb158db3b2dbebf747088 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 30 Jan 2025 06:56:23 +0000 Subject: [PATCH 32/58] fix failing test after refactor --- lib/app/star.ex | 2 +- lib/app/user.ex | 21 +++++++++++---------- lib/app_web/live/app_live.ex | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/app/star.ex b/lib/app/star.ex index 3bdfb52..4b2e0eb 100644 --- a/lib/app/star.ex +++ b/lib/app/star.ex @@ -40,7 +40,7 @@ defmodule App.Star do # We have multiple repos over 1k stars # Therefore issuing all these requests at once # would instantly hit the 5k/h GitHub API Request Limit - App.User.get_user_from_api(user) + App.User.create_user_with_hex(user) {:ok, star} = create(%{ user_id: user.id, repo_id: repo_id }) diff --git a/lib/app/user.ex b/lib/app/user.ex index ab468cc..0ed9e44 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -46,36 +46,37 @@ defmodule App.User do data = App.GitHub.user(user.login) # Not super happy about this crude error handling ... feel free to refactor. if Map.has_key?(data, :status) && data.status == "404" do - # IO.inspect(" - - - - - - - - - - - - - - - - - #{user.login}") - # dbg(user) - # IO.inspect(" - - - - - - - - - - - - - - - - - - - - - - - ") {:ok, user} = dummy_data(user) |> create() user else - {:ok, user} = + create_user_with_hex(data) + end + end + + def create_user_with_hex(data) do + {:ok, user} = map_github_user_fields_to_table(data) |> Map.put(:hex, App.Img.get_avatar_color(data.avatar_url)) |> create() - user - end + user end # tidy data before insertion def map_github_user_fields_to_table(u) do Map.merge(u, %{ avatar_url: String.split(u.avatar_url, "?") |> List.first, - company: clean_company(u.company), + company: clean_company(u), }) end - def clean_company(company) do + def clean_company(u) do # avoid `String.replace(nil, "@", "", [])` error - if company == nil do + if not Map.has_key?(u, :company) or u.company == nil do "" else - String.replace(company, "@", "") + String.replace(u.company, "@", "") end end diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 80fb131..8f9adc2 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -48,7 +48,7 @@ defmodule AppWeb.AppLive do App.Repository.get_org_repos(org) # get all stargazers for a given repo |> Enum.map(fn repo -> - # dbg(repo) + dbg(repo) # App.Star.get_stargazers_for_repo(owner, repo) repo end) From 665c9736938c9b2dfdebf73b027437bda0caa2b0 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 31 Jan 2025 05:53:52 +0000 Subject: [PATCH 33/58] create list_incomplete_users/0 for #226 --- lib/app/github.ex | 8 ++----- lib/app/star.ex | 8 +++---- lib/app/user.ex | 26 ++++++++++++++++----- lib/app_web/live/app_live.ex | 6 ++--- lib/app_web/live/app_live.html.heex | 8 ++++--- lib/app_web/templates/layout/root.html.heex | 7 ++++-- test/app/github_test.exs | 2 +- test/app/star_test.exs | 2 +- test/app/user_test.exs | 10 +++++++- 9 files changed, 50 insertions(+), 27 deletions(-) diff --git a/lib/app/github.ex b/lib/app/github.ex index 1b77b6c..ceb028e 100644 --- a/lib/app/github.ex +++ b/lib/app/github.ex @@ -14,7 +14,6 @@ defmodule App.GitHub do """ def repository(owner, reponame) do log("repository", "#{owner}/#{reponame}") - # Logger.info "Fetching repository #{owner}/#{reponame}" {_status, data, _res} = Tentacat.Repositories.repo_get(@client, owner, reponame) data @@ -25,7 +24,6 @@ defmodule App.GitHub do """ def org_repos(owner) do log("org_repos", owner) - # Logger.info "Fetching list of repositories for #{owner}" {_status, data, _res} = Tentacat.Repositories.list_orgs(@client, owner) data @@ -35,7 +33,6 @@ defmodule App.GitHub do `user/1` Returns the GitHub user profile data. """ def user(username) do - # Logger.info "Fetching user #{username}" log("user", username) {_status, data, _res} = Tentacat.Users.find(@client, username) data @@ -45,7 +42,6 @@ defmodule App.GitHub do `org_user_list/1` Returns the list of GitHub users for an org. """ def org_user_list(orgname) do - # Logger.info "Fetching org user list for #{orgname}" log("org_user_list", orgname) {_status, data, _res} = Tentacat.Organizations.Members.list(@client, orgname) @@ -57,9 +53,9 @@ defmodule App.GitHub do `owner` - the owner of the repo `repo` - name of the repo to check stargazers for. """ - def repo_stargazers(owner, repo) do + def repo_stargazers(fullname) do + [owner, repo] = String.split(fullname, "/") log("repo_stargazers", "#{owner}/#{repo}") - # Logger.info "Fetching stargazers for #{owner}/#{repo}" {_status, data, _res} = Tentacat.Users.Starring.stargazers(@client, owner, repo) data diff --git a/lib/app/star.ex b/lib/app/star.ex index 4b2e0eb..6afb276 100644 --- a/lib/app/star.ex +++ b/lib/app/star.ex @@ -33,14 +33,14 @@ defmodule App.Star do `get_stargazers_for_repo/2` gets the starts for a given `owner` and `repo` inserts any new `users`. """ - def get_stargazers_for_repo(owner, repo) do - repo_id = App.Repository.get_repo_id_by_full_name("#{owner}/#{repo}") - App.GitHub.repo_stargazers(owner, repo) + def get_stargazers_for_repo(fullname) do + repo_id = App.Repository.get_repo_id_by_full_name(fullname) + App.GitHub.repo_stargazers(fullname) |> Enum.map(fn user -> # We have multiple repos over 1k stars # Therefore issuing all these requests at once # would instantly hit the 5k/h GitHub API Request Limit - App.User.create_user_with_hex(user) + App.User.create_incomplete_user_no_overwrite(user) {:ok, star} = create(%{ user_id: user.id, repo_id: repo_id }) diff --git a/lib/app/user.ex b/lib/app/user.ex index 0ed9e44..53e3b7f 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -63,6 +63,20 @@ defmodule App.User do user end + # This is useful when inserting partial user records e.g. stargazers + def create_incomplete_user_no_overwrite(data) do + partial_data = + map_github_user_fields_to_table(data) + |> Map.put(:hex, App.Img.get_avatar_color(data.avatar_url)) + + {:ok, user} = + %User{} + |> changeset(partial_data) + |> Repo.insert(on_conflict: :nothing, conflict_target: [:id]) + + user + end + # tidy data before insertion def map_github_user_fields_to_table(u) do Map.merge(u, %{ @@ -101,12 +115,12 @@ defmodule App.User do }, u) end - # def list_users do - # User - # |> limit(20) - # |> order_by(desc: :inserted_at) - # |> Repo.all() - # end + def list_incomplete_users do + from(u in User, select: %{login: u.login}, where: is_nil(u.created_at)) + |> limit(80) + |> order_by(desc: :inserted_at) + |> Repo.all() + end def list_users_avatars do from(u in User, select: %{id: u.id}) diff --git a/lib/app_web/live/app_live.ex b/lib/app_web/live/app_live.ex index 8f9adc2..334a0a5 100644 --- a/lib/app_web/live/app_live.ex +++ b/lib/app_web/live/app_live.ex @@ -48,9 +48,9 @@ defmodule AppWeb.AppLive do App.Repository.get_org_repos(org) # get all stargazers for a given repo |> Enum.map(fn repo -> - dbg(repo) - # App.Star.get_stargazers_for_repo(owner, repo) - repo + # dbg(repo) + App.Star.get_stargazers_for_repo(repo.full_name) + # repo end) end) diff --git a/lib/app_web/live/app_live.html.heex b/lib/app_web/live/app_live.html.heex index 4775d3b..b9905f3 100644 --- a/lib/app_web/live/app_live.html.heex +++ b/lib/app_web/live/app_live.html.heex @@ -4,12 +4,14 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round

{@count}

-
+

Newest person:

- + + {@data.name}
@@ -40,7 +42,7 @@ class="w-20 bg-green-600 hover:bg-green-800 text-white font-bold py-2 px-4 round
-
+
diff --git a/lib/app_web/templates/layout/root.html.heex b/lib/app_web/templates/layout/root.html.heex index 9d2a250..be8778a 100644 --- a/lib/app_web/templates/layout/root.html.heex +++ b/lib/app_web/templates/layout/root.html.heex @@ -12,7 +12,7 @@ - + - - <%= @inner_content %> +
+ <%= @inner_content %> +
diff --git a/test/app/github_test.exs b/test/app/github_test.exs index 09d5d5e..2f4a091 100644 --- a/test/app/github_test.exs +++ b/test/app/github_test.exs @@ -46,9 +46,16 @@ defmodule App.GitHubTest do test "App.GitHub.org_user_list/1" do orgname = "ideaq" list = GitHub.org_user_list(orgname) + dbg(list) assert length(list) > 2 end + test "App.GitHub.org_user_list/1 UNHAPPY PATH issue #244" do + orgname = "2024-hgu-ccd-one-in-christ" + list = GitHub.org_user_list(orgname) + dbg(list) + end + test "App.GitHub.user/1 known 404 (unhappy path)" do username = "kittenking" data = App.GitHub.user(username) From c639a5e658a960cc8d20a0154e5c82e20f1ec46a Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 25 Feb 2025 11:57:32 +0000 Subject: [PATCH 54/58] 3 failing tests ... #244 --- lib/app/org.ex | 53 +++++++++++++++++-- lib/app/orgmember.ex | 19 ++++--- lib/app/user.ex | 2 +- .../20221005213110_create_users.exs | 4 +- .../migrations/20240608112859_create_orgs.exs | 5 +- test/app/org_test.exs | 12 +++++ 6 files changed, 81 insertions(+), 14 deletions(-) diff --git a/lib/app/org.ex b/lib/app/org.ex index 295ca7f..cb80792 100644 --- a/lib/app/org.ex +++ b/lib/app/org.ex @@ -14,7 +14,7 @@ defmodule App.Org do field :followers, :integer field :hex, :string field :location, :string - field :login, :string + field :login, :string #, primary_key: true field :name, :string field :public_repos, :integer field :show, :boolean, default: false @@ -27,7 +27,8 @@ defmodule App.Org do org |> cast(attrs, [:id, :login, :avatar_url, :hex, :description, :name, :company, :created_at, :public_repos, :location, :followers, :show]) - |> validate_required([:id, :login, :avatar_url]) + # |> validate_required([:id, :login, :avatar_url]) + # |> unique_constraint(:login, name: :org_login_unique) end @doc """ @@ -49,8 +50,7 @@ defmodule App.Org do data = App.GitHub.org(org.login) # Not super happy about this crude error handling ... feel free to refactor. if Map.has_key?(data, :status) && data.status == "404" do - # do nothing - org + update_org_created(org) else create_org_with_hex(data) end @@ -91,4 +91,49 @@ defmodule App.Org do App.Follow.get_followers_from_api(org.login, true) end) end + + @doc """ + `update_org_created/1` updates the `created_at` date to Now + so that we don't keep requesting the data from GitHub. + """ + def update_org_created(org) do + org = get_or_create_org(org) |> dbg() + {:ok, org_updated} = + Ecto.Changeset.change(org, %{created_at: now()}) + |> App.Repo.update() + + # return the org that was updated: + org_updated + end + + @doc """ + `now/0` returns a `NaiveDateTime` as a `String` in format: 2025-02-25 07:29:10 + """ + def now do + NaiveDateTime.utc_now() + |> NaiveDateTime.to_string() + |> String.split(".") + |> List.first() + end + + def strip_struct_metadata(struct) do + struct + |> Map.delete(:__meta__) + |> Map.delete(:__struct__) + end + + + def get_or_create_org(org) do + org_data = App.Repo.get_by(App.Org, login: org.login) + org_data = if is_nil(org_data) do + dbg(org) + {:ok, org_data} = + create(Map.merge(org, %{id: :rand.uniform(1_000_000_000_000)})) + + org_data + else + org_data + end + strip_struct_metadata(org_data) + end end diff --git a/lib/app/orgmember.ex b/lib/app/orgmember.ex index ec162d6..47f27f9 100644 --- a/lib/app/orgmember.ex +++ b/lib/app/orgmember.ex @@ -50,14 +50,19 @@ defmodule App.Orgmember do """ def get_users_for_org(org) do # Get the list of orgs a user belongs to (public) - App.GitHub.org_user_list(org.login) - |> Enum.map(fn user -> - App.User.create_incomplete_user_no_overwrite(user) + data = App.GitHub.org_user_list(org.login) + if Map.has_key?(data, :status) && data.status == 404 do - # insert the orgmember record: - create(%{org_id: org.id, user_id: user.id}) + nil + else + data + |> Enum.map(fn user -> + App.User.create_incomplete_user_no_overwrite(user) + # insert the orgmember record: + create(%{org_id: org.id, user_id: user.id}) - user - end) + user + end) + end end end diff --git a/lib/app/user.ex b/lib/app/user.ex index fdf6028..e1730ad 100644 --- a/lib/app/user.ex +++ b/lib/app/user.ex @@ -15,7 +15,7 @@ defmodule App.User do field :followers, :integer field :following, :integer field :hex, :string - field :hireable, :boolean, default: false + field :hireable, :boolean #, default: false field :location, :string field :login, :string field :name, :string diff --git a/priv/repo/migrations/20221005213110_create_users.exs b/priv/repo/migrations/20221005213110_create_users.exs index 28a8464..c486b2b 100644 --- a/priv/repo/migrations/20221005213110_create_users.exs +++ b/priv/repo/migrations/20221005213110_create_users.exs @@ -14,12 +14,14 @@ defmodule App.Repo.Migrations.CreateUsers do add :hireable, :boolean, default: false add :hex, :string add :location, :string - add :login, :string + add :login, :string #, primary_key: true add :name, :string add :public_repos, :integer add :two_factor_authentication, :boolean, default: false timestamps() end + + # create unique_index(:users, :login, name: :login_unique) end end diff --git a/priv/repo/migrations/20240608112859_create_orgs.exs b/priv/repo/migrations/20240608112859_create_orgs.exs index 8861809..98553e5 100644 --- a/priv/repo/migrations/20240608112859_create_orgs.exs +++ b/priv/repo/migrations/20240608112859_create_orgs.exs @@ -11,12 +11,15 @@ defmodule App.Repo.Migrations.CreateOrgs do add :followers, :integer add :hex, :string add :location, :string - add :login, :string + add :login, :string #, primary_key: true add :name, :string add :public_repos, :integer add :show, :boolean, default: false timestamps() end + + # drop_if_exists index(:orgs, [:login]) + # create unique_index(:orgs, :login, name: :org_login_unique) end end diff --git a/test/app/org_test.exs b/test/app/org_test.exs index 8f3eb02..d0b80b8 100644 --- a/test/app/org_test.exs +++ b/test/app/org_test.exs @@ -57,4 +57,16 @@ defmodule App.OrgTest do assert updated_org.description == "a Q of Ideas" assert updated_org.created_at == "2014-03-02T13:18:11Z" end + + test "App.Org.update_org_created/1 updates the created_at date to now" do + {:ok, org} = App.Org.create(%{ + id: 6_831_072, + avatar_url: "https://avatars.githubusercontent.com/u/6831072", + login: "myawesomeorg" + }) + assert org.created_at == nil + now = App.Org.now() + updated_org = App.Org.update_org_created(org) + assert updated_org.created_at == now + end end From 466dbfa1a9dbbe48764d963980e9fed043464ab3 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 27 Feb 2025 11:25:17 +0000 Subject: [PATCH 55/58] fix faling tests (adds error handling for #244) --- lib/app/org.ex | 9 ++++----- lib/app/orgmember.ex | 4 ++-- mix.exs | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/app/org.ex b/lib/app/org.ex index cb80792..acaafd8 100644 --- a/lib/app/org.ex +++ b/lib/app/org.ex @@ -97,7 +97,7 @@ defmodule App.Org do so that we don't keep requesting the data from GitHub. """ def update_org_created(org) do - org = get_or_create_org(org) |> dbg() + org = get_or_create_org(org) {:ok, org_updated} = Ecto.Changeset.change(org, %{created_at: now()}) |> App.Repo.update() @@ -124,9 +124,9 @@ defmodule App.Org do def get_or_create_org(org) do - org_data = App.Repo.get_by(App.Org, login: org.login) - org_data = if is_nil(org_data) do - dbg(org) + org_data = get_org_by_login(org.login) + if is_nil(org_data) do + # dbg(org) {:ok, org_data} = create(Map.merge(org, %{id: :rand.uniform(1_000_000_000_000)})) @@ -134,6 +134,5 @@ defmodule App.Org do else org_data end - strip_struct_metadata(org_data) end end diff --git a/lib/app/orgmember.ex b/lib/app/orgmember.ex index 47f27f9..04306e6 100644 --- a/lib/app/orgmember.ex +++ b/lib/app/orgmember.ex @@ -51,9 +51,9 @@ defmodule App.Orgmember do def get_users_for_org(org) do # Get the list of orgs a user belongs to (public) data = App.GitHub.org_user_list(org.login) - if Map.has_key?(data, :status) && data.status == 404 do + if is_map(data) && Map.has_key?(data, :status) && data.status == 404 do - nil + [] else data |> Enum.map(fn user -> diff --git a/mix.exs b/mix.exs index 3364239..1994e76 100644 --- a/mix.exs +++ b/mix.exs @@ -111,8 +111,8 @@ defmodule App.MixProject do "phx.digest" ], test: ["ecto.reset", "test"], - t: ["test"], - c: ["coveralls.html"], + t: ["test --trace"], + c: ["coveralls.html --trace"], s: ["phx.server"] ] end From 22427385a4054c89d8121603e0e8b91ada8623c0 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Thu, 27 Feb 2025 11:32:51 +0000 Subject: [PATCH 56/58] comment-out unused code. 100% cov closes #244 --- lib/app/org.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/app/org.ex b/lib/app/org.ex index acaafd8..5802d74 100644 --- a/lib/app/org.ex +++ b/lib/app/org.ex @@ -116,11 +116,11 @@ defmodule App.Org do |> List.first() end - def strip_struct_metadata(struct) do - struct - |> Map.delete(:__meta__) - |> Map.delete(:__struct__) - end + # def strip_struct_metadata(struct) do + # struct + # |> Map.delete(:__meta__) + # |> Map.delete(:__struct__) + # end def get_or_create_org(org) do From 33177de3fc0cb7792f30a203151c7a6bcaad3d24 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 28 Feb 2025 11:56:09 +0000 Subject: [PATCH 57/58] document creation of stats schema for #233-n --- BUILDIT.md | 30 ++++++++++++++++++++++++++++++ mix.lock | 1 + 2 files changed, 31 insertions(+) diff --git a/BUILDIT.md b/BUILDIT.md index 2e301af..1c4df88 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -68,6 +68,7 @@ where (_hopefully_) it will all be clear. - [4.1 Limit API Requests](#41-limit-api-requests) - [5. Org \<-\> Users](#5-org---users) - [6. Repository Contributors](#6-repository-contributors) +- [7. Dashboard](#7-dashboard) - [X. Add Authentication](#x-add-authentication) - [X.1 Add `auth_plug` to `deps`](#x1-add-auth_plug-to-deps) - [X.2 Get your `AUTH_API_KEY`](#x2-get-your-auth_api_key) @@ -988,6 +989,35 @@ The following command: mix phx.gen.schema Contrib contribs repo_id:integer user_id:integer count:integer ``` +That creates the migration: +`priv/repo/migrations/20250203140127_create_contribs.exs` + +See:∫ +`lib/app/contrib.ex` for the code +and +`test/app/contrib_test.exs` +for tests. + +# 7. Dashboard + +We want to have a **`dashboard`** displaying the most important **stats**. +We have a list of these **stats** including: + ++ Total number of stars for the @dwyl org ⭐ + [#238](https://github.com/dwyl/who/issues/238) ++ List of top contributors πŸ” + [#237](https://github.com/dwyl/who/issues/238) ++ Number of `@dwy`l` org members πŸ‘€ + [#245](https://github.com/dwyl/who/issues/245) ++ ... many more to come! + +In order to speed up the rendering of the **`dashboard`** +and store the _history_ of each stat that we want to track, +we will have a schema for it which we can add to over time. + +```elixir +mix phx.gen.schema Stat stats stars:integer members:integer total_contribs:integer +``` diff --git a/mix.lock b/mix.lock index 523fbf9..8249fcc 100644 --- a/mix.lock +++ b/mix.lock @@ -3,6 +3,7 @@ "auth_plug": {:hex, :auth_plug, "1.5.0", "fa9f8c022d76cd7ef4cd322f25846698f18a0b6d2435a3ae1cb0e61167199e52", [:mix], [{:envar, "~> 1.0.8", [hex: :envar, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.8.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.5", [hex: :joken, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:useful, "~> 1.0.8", [hex: :useful, repo: "hexpm", optional: false]}], "hexpm", "3481aed63f8bb24081268db6713c931c33b5b3b17d2f53ca7949ff8443c1fbb7"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.11", "4bbd584741601eb658007339ea730b082cc61f3554cf2e8f39bf693a11b49073", [:mix], [], "hexpm", "e03990b4db988df56262852f20de0f659871c35154691427a5047f4967a16a62"}, + "cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "comeonin": {:hex, :comeonin, "5.5.0", "364d00df52545c44a139bad919d7eacb55abf39e86565878e17cebb787977368", [:mix], [], "hexpm", "6287fc3ba0aad34883cbe3f7949fc1d1e738e5ccdce77165bc99490aa69f47fb"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, From 06bcdc13f81aab54503b3659abb563ab1892c227 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sat, 8 Mar 2025 15:05:59 +0000 Subject: [PATCH 58/58] update actions/cache@v2 -> v4 --- .github/workflows/ci.yml | 3 ++- BUILDIT.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2645f4d..c3071d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,9 @@ jobs: with: elixir-version: '1.18.2' # Define the elixir version [required] otp-version: '27.2' # Define the OTP version [required] + # https://github.com/actions/cache - name: Restore dependencies cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} diff --git a/BUILDIT.md b/BUILDIT.md index 1c4df88..7b99946 100644 --- a/BUILDIT.md +++ b/BUILDIT.md @@ -1016,7 +1016,7 @@ and store the _history_ of each stat that we want to track, we will have a schema for it which we can add to over time. ```elixir -mix phx.gen.schema Stat stats stars:integer members:integer total_contribs:integer +mix phx.gen.schema Stat stats followers:integer members:integer total_stars:integer total_contribs:integer ```