Aug 27, 2018 · Sep 10, 2018 · Sep 17, 2018 · Sep 17, 2018 · Sep 24, 2018 · Sep 24, 2018
diff --git a/Makefile b/Makefile
 mix test.watch --only wip
 test.watch.wip2:
 mix test.watch --only wip2
 test.watch.bug:
 mix test.watch --only bug
 test.db_reset:
 env MIX_ENV=test mix ecto.drop
 env MIX_ENV=test mix ecto.create
diff --git a/Makefile.include.mk b/Makefile.include.mk
 @echo "  ....................................................."
 @echo "  test.watch.wip   : run @wip test in watch mode"
 @echo "  ....................................................."
 @echo "  test.watch.wip2  : shortcut for lots of @wip around"
 @echo "  ....................................................."
 @echo "  test.watch.bug  : sometimes fails for unkown reason"
 @echo "  ....................................................."
 @echo "  test.db_reset    : reset test database"
 @echo "                   | needed when add new migration"
 @echo "  ....................................................."
diff --git a/README.md b/README.md

 ![CPS Brand](https://github.com/mydearxym/mastani_server/blob/dev/docs/snapshots/cps_logo_md.png)
 ![CPS Brand](https://github.com/mydearxym/mastani_server/blob/dev/docs/snapshots/cps_logo_md.png?raw=true)
 =========
 [![Build Status](https://travis-ci.org/coderplanets/coderplanets_server.svg?branch=dev)](https://travis-ci.org/coderplanets/coderplanets_server)
 [![codecov](https://codecov.io/gh/coderplanets/coderplanets_server/branch/dev/graph/badge.svg)](https://codecov.io/gh/coderplanets/coderplanets_server)
diff --git a/config/config.exs b/config/config.exs
  metadata: [:request_id]

 config :phoenix, :format_encoders, json: Jason
 config :ecto, json_library: Jason

 # TODO move this config to secret later
 config :mastani_server, Helper.Guardian,
  user_achieve_favorite_weight: 2,
  user_achieve_follow_weight: 3

 config :mastani_server, :customization,
  theme: "cyan",
  community_chart: false,
  brainwash_free: false,
  banner_layout: "digest",
  contents_layout: "digest",
  content_divider: false,
  mark_viewed: true,
  display_density: "20"

 config :mastani_server, MastaniServerWeb.Gettext, default_locale: "zh_CN", locales: ~w(en zh_CN)

 import_config "#{Mix.env()}.exs"
diff --git a/deploy/production/api_server.tar.gz b/deploy/production/api_server.tar.gz
diff --git a/lib/helper/PublicIpPlug.ex b/lib/helper/PublicIpPlug.ex
 # https://www.cogini.com/blog/getting-the-client-public-ip-address-in-phoenix/
 defmodule Helper.PublicIpPlug do
  @moduledoc "Get public IP address of request from x-forwarded-for header"
  @behaviour Plug
  @app :mastani_server

  def init(opts), do: opts

  def call(%{assigns: %{ip: _}} = conn, _opts), do: conn

  def call(conn, _opts) do
    process(conn, Plug.Conn.get_req_header(conn, "x-forwarded-for"))
  end

  def process(conn, []) do
    Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(get_peer_ip(conn))))
  end

  def process(conn, vals) do
    if Application.get_env(@app, :trust_x_forwarded_for, false) do
      ip_address = get_ip_address(conn, vals)

      # Rewrite standard remote_ip field with value from header
      # See https://hexdocs.pm/plug/Plug.Conn.html
      conn = %{conn | remote_ip: ip_address}

      Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(ip_address)))
    else
      Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(get_peer_ip(conn))))
    end
  end

  defp get_ip_address(conn, vals)
  defp get_ip_address(conn, []), do: get_peer_ip(conn)

  defp get_ip_address(conn, [val | _]) do
    # Split into multiple values
    comps =
      val
      |> String.split(~r{\s*,\s*}, trim: true)
      # Get rid of "unknown" values
      |> Enum.filter(&(&1 != "unknown"))
      # Split IP from port, if any
      |> Enum.map(&hd(String.split(&1, ":")))
      # Filter out blanks
      |> Enum.filter(&(&1 != ""))
      # Parse address into :inet.ip_address tuple
      |> Enum.map(&parse_address(&1))
      # Elminate internal IP addreses, e.g. 192.168.1.1
      |> Enum.filter(&is_public_ip(&1))

    case comps do
      [] -> get_peer_ip(conn)
      [comp | _] -> comp
    end
  end

  @spec get_peer_ip(Plug.Conn.t()) :: :inet.ip_address()
  defp get_peer_ip(conn) do
    {ip, _port} = conn.peer
    ip
  end

  @spec parse_address(String.t()) :: :inet.ip_address()
  defp parse_address(ip) do
    case :inet.parse_ipv4strict_address(to_charlist(ip)) do
      {:ok, ip_address} -> ip_address
      {:error, :einval} -> :einval
    end
  end

  # Whether the input is a valid, public IP address
  # http://en.wikipedia.org/wiki/Private_network
  @spec is_public_ip(:inet.ip_address() | atom) :: boolean
  defp is_public_ip(ip_address) do
    case ip_address do
      {10, _, _, _} -> false
      {192, 168, _, _} -> false
      {172, second, _, _} when second >= 16 and second <= 31 -> false
      {127, 0, 0, _} -> false
      {_, _, _, _} -> true
      :einval -> false
    end
  end
 end
diff --git a/lib/helper/query_builder.ex b/lib/helper/query_builder.ex
    |> select([f], count(f.id))
  end

  def members_pack(queryable, %{count: _, type: :job}) do
    queryable
    |> group_by([f], f.job_id)
    |> select([f], count(f.id))
  end

  def members_pack(queryable, %{count: _, type: :video}) do
    queryable
    |> group_by([f], f.video_id)
    |> select([f], count(f.id))
  end

  def members_pack(queryable, %{count: _, type: :repo}) do
    queryable
    |> group_by([f], f.repo_id)
    |> select([f], count(f.id))
  end

  def members_pack(queryable, %{count: _, type: :community}) do
    queryable
    |> group_by([f], f.community_id)
          where: t.title == ^tag_name
        )

      {:topic, topic}, queryable ->
        from(
          q in queryable,
          join: t in assoc(q, :topics),
          where: t.raw == ^topic
        )

      {:category, catetory_raw}, queryable ->
        from(
          q in queryable,
          where: t.raw == ^community_raw
        )

      {:one_community, community_raw}, queryable ->
        from(
          q in queryable,
          join: t in assoc(q, :community),
          where: t.raw == ^community_raw
        )

      {:first, first}, queryable ->
        queryable |> limit(^first)

diff --git a/lib/helper/radar_search.ex b/lib/helper/radar_search.ex
 defmodule Helper.RadarSearch do
  @moduledoc """
  find city info by ip
  refer: https://lbs.amap.com/api/webservice/guide/api/ipconfig/?sug_index=0
  """
  use Tesla, only: [:get]
  import Helper.Utils, only: [get_config: 2]

  @endpoint "https://restapi.amap.com/v3/ip"
  @ip_service_key get_config(:radar_search, :ip_service)
  @timeout_limit 5000

  # plug(Tesla.Middleware.BaseUrl, "https://restapi.amap.com/v3/ip")
  plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
  plug(Tesla.Middleware.Timeout, timeout: @timeout_limit)
  plug(Tesla.Middleware.JSON)

  @doc """
  this is only match fail situation in test
  """
  def locate_city(ip \\ "14.196.0.0")

  def locate_city(:fake_ip) do
    {:error, "not found"}
  end

  # http://ip.yqie.com/search.aspx?searchword=%E6%88%90%E9%83%BD%E5%B8%82
  def locate_city(ip) do
    query = [ip: ip, key: @ip_service_key]

    with true <- Mix.env() !== :test do
      case get(@endpoint, query: query) do
        %{status: 200, body: body} ->
          handle_result({:ok, body["city"]})

        _ ->
          {:error, "error"}
      end
    else
      _ ->
        {:ok, "成都"}
        # {:error, "error"}
    end
  end

  defp handle_result({:ok, result}) do
    case result do
      [] -> {:error, "not found"}
      _ -> cut_tail({:ok, result})
    end
  end

  defp cut_tail({:ok, result}) do
    case String.last(result) == "市" do
      true -> {:ok, String.trim_trailing(result, "市")}
      false -> {:ok, result}
    end
  end
 end
diff --git a/lib/helper/utils.ex b/lib/helper/utils.ex
  import Helper.ErrorHandler
  import Helper.ErrorCode

  def get_config(section, key, app \\ :mastani_server) do
  def get_config(section, key, app \\ :mastani_server)

  def get_config(section, :all, app) do
    app
    |> Application.get_env(section)
    |> case do
      nil -> ""
      config -> config
    end
  end

  def get_config(section, key, app) do
    app
    |> Application.get_env(section)
    # |> IO.inspect(label: "debug ci")
    |> case do
      nil -> ""
      config -> Keyword.get(config, key)
  def done(_, :boolean), do: {:ok, true}
  def done(nil, err_msg), do: {:error, err_msg}
  def done({:ok, _}, with: result), do: {:ok, result}
  def done({:error, error}, with: _result), do: {:error, error}

  def done({:ok, %{id: id}}, :status), do: {:ok, %{done: true, id: id}}
  def done({:error, _}, :status), do: {:ok, %{done: false}}
    map |> Enum.reduce(%{}, fn {key, val}, acc -> Map.put(acc, to_string(key), val) end)
  end

  @doc """
  Recursivly camelize the map keys
  usage: convert factory attrs to used for simu Graphql parmas
  """
  def camelize_map_key(map) do
    map_list =
      Enum.map(map, fn {k, v} ->
        v =
          cond do
            is_datetime?(v) ->
              DateTime.to_iso8601(v)

            is_map(v) ->
              camelize_map_key(safe_map(v))

            true ->
              v
          end

        map_to_camel({k, v})
      end)

    Enum.into(map_list, %{})
  end

  defp safe_map(%{__struct__: _} = map), do: Map.from_struct(map)
  defp safe_map(map), do: map

  defp map_to_camel({k, v}), do: {Recase.to_camel(to_string(k)), v}

  def is_datetime?(%DateTime{}), do: true
  def is_datetime?(_), do: false

  def deep_merge(left, right) do
    Map.merge(left, right, &deep_resolve/3)
  end
  def map_atom_value(attrs, :string) do
    results =
      Enum.map(attrs, fn {k, v} ->
        if is_atom(v) do
          {k, to_string(v)}
        else
          {k, v}
        cond do
          v == true or v == false ->
            {k, v}

          is_atom(v) ->
            {k, v |> to_string() |> String.downcase()}

          true ->
            {k, v}
        end
      end)

  # NOT a map. We fall back to standard merge behavior, preferring
  # the value on the right.
  defp deep_resolve(_key, _left, right), do: right

  @doc """
  ["a", "b", "c", "c"] => %{"a" => 1, "b" => 1, "c" => 2}
  """
  def count_words(words) when is_list(words) do
    Enum.reduce(words, %{}, &update_word_count/2)
  end

  defp update_word_count(word, acc) do
    Map.update(acc, to_string(word), 1, &(&1 + 1))
  end
 end
Original file line number	Diff line number	Diff line change
Expand Up		@@ -125,6 +125,8 @@ test.watch.wip:
		mix test.watch --only wip
		test.watch.wip2:
		mix test.watch --only wip2
		test.watch.bug:
		mix test.watch --only bug
		test.db_reset:
		env MIX_ENV=test mix ecto.drop
		env MIX_ENV=test mix ecto.create
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -130,6 +130,10 @@ define test.help
		@echo " ....................................................."
		@echo " test.watch.wip : run @wip test in watch mode"
		@echo " ....................................................."
		@echo " test.watch.wip2 : shortcut for lots of @wip around"
		@echo " ....................................................."
		@echo " test.watch.bug : sometimes fails for unkown reason"
		@echo " ....................................................."
		@echo " test.db_reset : reset test database"
		@echo " \| needed when add new migration"
		@echo " ....................................................."
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,5 +1,5 @@

		![CPS Brand](https://github.com/mydearxym/mastani_server/blob/dev/docs/snapshots/cps_logo_md.png)
		![CPS Brand](https://github.com/mydearxym/mastani_server/blob/dev/docs/snapshots/cps_logo_md.png?raw=true)
		=========
		[![Build Status](https://travis-ci.org/coderplanets/coderplanets_server.svg?branch=dev)](https://travis-ci.org/coderplanets/coderplanets_server)
		[![codecov](https://codecov.io/gh/coderplanets/coderplanets_server/branch/dev/graph/badge.svg)](https://codecov.io/gh/coderplanets/coderplanets_server)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -21,6 +21,7 @@ config :logger, :console,
		metadata: [:request_id]

		config :phoenix, :format_encoders, json: Jason
		config :ecto, json_library: Jason

		# TODO move this config to secret later
		config :mastani_server, Helper.Guardian,
Expand DownExpand Up		@@ -50,6 +51,16 @@ config :mastani_server, :general,
		user_achieve_favorite_weight: 2,
		user_achieve_follow_weight: 3

		config :mastani_server, :customization,
		theme: "cyan",
		community_chart: false,
		brainwash_free: false,
		banner_layout: "digest",
		contents_layout: "digest",
		content_divider: false,
		mark_viewed: true,
		display_density: "20"

		config :mastani_server, MastaniServerWeb.Gettext, default_locale: "zh_CN", locales: ~w(en zh_CN)

		import_config "#{Mix.env()}.exs"
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,85 @@
		# https://www.cogini.com/blog/getting-the-client-public-ip-address-in-phoenix/
		defmodule Helper.PublicIpPlug do
		@moduledoc "Get public IP address of request from x-forwarded-for header"
		@behaviour Plug
		@app :mastani_server

		def init(opts), do: opts

		def call(%{assigns: %{ip: _}} = conn, _opts), do: conn

		def call(conn, _opts) do
		process(conn, Plug.Conn.get_req_header(conn, "x-forwarded-for"))
		end

		def process(conn, []) do
		Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(get_peer_ip(conn))))
		end

		def process(conn, vals) do
		if Application.get_env(@app, :trust_x_forwarded_for, false) do
		ip_address = get_ip_address(conn, vals)

		# Rewrite standard remote_ip field with value from header
		# See https://hexdocs.pm/plug/Plug.Conn.html
		conn = %{conn \| remote_ip: ip_address}

		Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(ip_address)))
		else
		Plug.Conn.assign(conn, :ip, to_string(:inet.ntoa(get_peer_ip(conn))))
		end
		end

		defp get_ip_address(conn, vals)
		defp get_ip_address(conn, []), do: get_peer_ip(conn)

		defp get_ip_address(conn, [val \| _]) do
		# Split into multiple values
		comps =
		val
		\|> String.split(~r{\s,\s}, trim: true)
		# Get rid of "unknown" values
		\|> Enum.filter(&(&1 != "unknown"))
		# Split IP from port, if any
		\|> Enum.map(&hd(String.split(&1, ":")))
		# Filter out blanks
		\|> Enum.filter(&(&1 != ""))
		# Parse address into :inet.ip_address tuple
		\|> Enum.map(&parse_address(&1))
		# Elminate internal IP addreses, e.g. 192.168.1.1
		\|> Enum.filter(&is_public_ip(&1))

		case comps do
		[] -> get_peer_ip(conn)
		[comp \| _] -> comp
		end
		end

		@spec get_peer_ip(Plug.Conn.t()) :: :inet.ip_address()
		defp get_peer_ip(conn) do
		{ip, _port} = conn.peer
		ip
		end

		@spec parse_address(String.t()) :: :inet.ip_address()
		defp parse_address(ip) do
		case :inet.parse_ipv4strict_address(to_charlist(ip)) do
		{:ok, ip_address} -> ip_address
		{:error, :einval} -> :einval
		end
		end

		# Whether the input is a valid, public IP address
		# http://en.wikipedia.org/wiki/Private_network
		@spec is_public_ip(:inet.ip_address() \| atom) :: boolean
		defp is_public_ip(ip_address) do
		case ip_address do
		{10, _, _, _} -> false
		{192, 168, _, _} -> false
		{172, second, _, _} when second >= 16 and second <= 31 -> false
		{127, 0, 0, _} -> false
		{_, _, _, _} -> true
		:einval -> false
		end
		end
		end
Original file line number	Diff line number	Diff line change
Expand Up		@@ -27,6 +27,24 @@ defmodule Helper.QueryBuilder do
		\|> select([f], count(f.id))
		end

		def members_pack(queryable, %{count: _, type: :job}) do
		queryable
		\|> group_by([f], f.job_id)
		\|> select([f], count(f.id))
		end

		def members_pack(queryable, %{count: _, type: :video}) do
		queryable
		\|> group_by([f], f.video_id)
		\|> select([f], count(f.id))
		end

		def members_pack(queryable, %{count: _, type: :repo}) do
		queryable
		\|> group_by([f], f.repo_id)
		\|> select([f], count(f.id))
		end

		def members_pack(queryable, %{count: _, type: :community}) do
		queryable
		\|> group_by([f], f.community_id)
Expand DownExpand Up		@@ -167,6 +185,13 @@ defmodule Helper.QueryBuilder do
		where: t.title == ^tag_name
		)

		{:topic, topic}, queryable ->
		from(
		q in queryable,
		join: t in assoc(q, :topics),
		where: t.raw == ^topic
		)

		{:category, catetory_raw}, queryable ->
		from(
		q in queryable,
Expand All		@@ -181,6 +206,13 @@ defmodule Helper.QueryBuilder do
		where: t.raw == ^community_raw
		)

		{:one_community, community_raw}, queryable ->
		from(
		q in queryable,
		join: t in assoc(q, :community),
		where: t.raw == ^community_raw
		)

		{:first, first}, queryable ->
		queryable \|> limit(^first)

Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,59 @@
		defmodule Helper.RadarSearch do
		@moduledoc """
		find city info by ip
		refer: https://lbs.amap.com/api/webservice/guide/api/ipconfig/?sug_index=0
		"""
		use Tesla, only: [:get]
		import Helper.Utils, only: [get_config: 2]

		@endpoint "https://restapi.amap.com/v3/ip"
		@ip_service_key get_config(:radar_search, :ip_service)
		@timeout_limit 5000

		# plug(Tesla.Middleware.BaseUrl, "https://restapi.amap.com/v3/ip")
		plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
		plug(Tesla.Middleware.Timeout, timeout: @timeout_limit)
		plug(Tesla.Middleware.JSON)

		@doc """
		this is only match fail situation in test
		"""
		def locate_city(ip \\ "14.196.0.0")

		def locate_city(:fake_ip) do
		{:error, "not found"}
		end

		# http://ip.yqie.com/search.aspx?searchword=%E6%88%90%E9%83%BD%E5%B8%82
		def locate_city(ip) do
		query = [ip: ip, key: @ip_service_key]

		with true <- Mix.env() !== :test do
		case get(@endpoint, query: query) do
		%{status: 200, body: body} ->
		handle_result({:ok, body["city"]})

		_ ->
		{:error, "error"}
		end
		else
		_ ->
		{:ok, "成都"}
		# {:error, "error"}
		end
		end

		defp handle_result({:ok, result}) do
		case result do
		[] -> {:error, "not found"}
		_ -> cut_tail({:ok, result})
		end
		end

		defp cut_tail({:ok, result}) do
		case String.last(result) == "市" do
		true -> {:ok, String.trim_trailing(result, "市")}
		false -> {:ok, result}
		end
		end
		end
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,10 +6,20 @@ defmodule Helper.Utils do
		import Helper.ErrorHandler
		import Helper.ErrorCode

		def get_config(section, key, app \\ :mastani_server) do
		def get_config(section, key, app \\ :mastani_server)

		def get_config(section, :all, app) do
		app
		\|> Application.get_env(section)
		\|> case do
		nil -> ""
		config -> config
		end
		end

		def get_config(section, key, app) do
		app
		\|> Application.get_env(section)
		# \|> IO.inspect(label: "debug ci")
		\|> case do
		nil -> ""
		config -> Keyword.get(config, key)
Expand All		@@ -23,6 +33,7 @@ defmodule Helper.Utils do
		def done(_, :boolean), do: {:ok, true}
		def done(nil, err_msg), do: {:error, err_msg}
		def done({:ok, _}, with: result), do: {:ok, result}
		def done({:error, error}, with: _result), do: {:error, error}

		def done({:ok, %{id: id}}, :status), do: {:ok, %{done: true, id: id}}
		def done({:error, _}, :status), do: {:ok, %{done: false}}
Expand DownExpand Up		@@ -69,6 +80,39 @@ defmodule Helper.Utils do
		map \|> Enum.reduce(%{}, fn {key, val}, acc -> Map.put(acc, to_string(key), val) end)
		end

		@doc """
		Recursivly camelize the map keys
		usage: convert factory attrs to used for simu Graphql parmas
		"""
		def camelize_map_key(map) do
		map_list =
		Enum.map(map, fn {k, v} ->
		v =
		cond do
		is_datetime?(v) ->
		DateTime.to_iso8601(v)

		is_map(v) ->
		camelize_map_key(safe_map(v))

		true ->
		v
		end

		map_to_camel({k, v})
		end)

		Enum.into(map_list, %{})
		end

		defp safe_map(%{__struct__: _} = map), do: Map.from_struct(map)
		defp safe_map(map), do: map

		defp map_to_camel({k, v}), do: {Recase.to_camel(to_string(k)), v}

		def is_datetime?(%DateTime{}), do: true
		def is_datetime?(_), do: false

		def deep_merge(left, right) do
		Map.merge(left, right, &deep_resolve/3)
		end
Expand All		@@ -87,10 +131,15 @@ defmodule Helper.Utils do
		def map_atom_value(attrs, :string) do
		results =
		Enum.map(attrs, fn {k, v} ->
		if is_atom(v) do
		{k, to_string(v)}
		else
		{k, v}
		cond do
		v == true or v == false ->
		{k, v}

		is_atom(v) ->
		{k, v \|> to_string() \|> String.downcase()}

		true ->
		{k, v}
		end
		end)

Expand All		@@ -106,4 +155,15 @@ defmodule Helper.Utils do
		# NOT a map. We fall back to standard merge behavior, preferring
		# the value on the right.
		defp deep_resolve(_key, _left, right), do: right

		@doc """
		["a", "b", "c", "c"] => %{"a" => 1, "b" => 1, "c" => 2}
		"""
		def count_words(words) when is_list(words) do
		Enum.reduce(words, %{}, &update_word_count/2)
		end

		defp update_word_count(word, acc) do
		Map.update(acc, to_string(word), 1, &(&1 + 1))
		end
		end