Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings
This repository was archived by the owner on Nov 8, 2022. It is now read-only.

feat: rss parse and basic workflow#431

Merged
mydearxym merged 6 commits intodevfromrss-parser
Sep 30, 2021
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletionsconfig/config.exs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -134,10 +134,24 @@ config :groupher_server, GroupherServer.Mailer,
adapter: Bamboo.MailgunAdapter,
domain: "mailer.coderplanets.com"

# handle background jobs
config :rihanna,
jobs_table_name: "background_jobs",
producer_postgres_connection: {Ecto, GroupherServer.Repo}
config :groupher_server, :cache,
pool: %{
common: %{
name: :common,
size: 5000,
minutes: 10
},
user_login: %{
name: :user_login,
size: 10_000,
minutes: 10_080
},
blog_rss: %{
name: :blog_rss,
size: 1000,
minutes: 15
}
}

# cron-like job scheduler
config :groupher_server, Helper.Scheduler,
Expand All@@ -147,6 +161,11 @@ config :groupher_server, Helper.Scheduler,
{"@daily", {Helper.Scheduler, :archive_artiments, []}}
]

# handle background jobs
config :rihanna,
jobs_table_name: "background_jobs",
producer_postgres_connection: {Ecto, GroupherServer.Repo}

import_config "#{Mix.env()}.exs"

if File.exists?("config/#{Mix.env()}.secret.exs") do
Expand Down
48 changes: 32 additions & 16 deletionslib/groupher_server/application.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,31 @@
defmodule GroupherServer.Application do
@moduledoc false
use Application
import Helper.Utils, only: [get_config: 2]

alias Helper.Cache

@cache_pool get_config(:cache, :pool)

# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
@spec start(any, any) :: {:error, any} | {:ok, pid}
def start(_type, _args) do
import Supervisor.Spec
alias Helper.Cache

# Define workers and child supervisors to be supervised
children = [
# Start the PubSub system
{Phoenix.PubSub, name: MyApp.PubSub},
# Start the Ecto repository
supervisor(GroupherServer.Repo, []),
# Start the endpoint when the application starts
supervisor(GroupherServerWeb.Endpoint, []),
# Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3)
# worker(GroupherServer.Worker, [arg1, arg2, arg3]),
worker(Cachex, [:common, Cache.config(:common)], id: :common),
worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login),
#
worker(Helper.Scheduler, []),
{Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]}
]
children =
[
# Start the PubSub system
{Phoenix.PubSub, name: MyApp.PubSub},
# Start the Ecto repository
supervisor(GroupherServer.Repo, []),
# Start the endpoint when the application starts
supervisor(GroupherServerWeb.Endpoint, []),
# Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3)
worker(Helper.Scheduler, []),
{Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]}
] ++ cache_workers()

# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
Expand All@@ -38,4 +39,19 @@ defmodule GroupherServer.Application do
GroupherServerWeb.Endpoint.config_change(changed, removed)
:ok
end

defp cache_workers() do
import Supervisor.Spec

# worker(GroupherServer.Worker, [arg1, arg2, arg3]),
# worker(Cachex, [:common, Cache.config(:common)], id: :common),
# worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login),
# worker(Cachex, [:blog_rss, Cache.config(:blog_rss)], id: :blog_rss),
@cache_pool
|> Map.keys()
|> Enum.reduce([], fn key, acc ->
name = @cache_pool[key].name
acc ++ [worker(Cachex, [name, Cache.config(key)], id: name)]
end)
end
end
6 changes: 6 additions & 0 deletionslib/groupher_server/cms/cms.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,7 @@ defmodule GroupherServer.CMS do
alias Delegate.{
AbuseReport,
ArticleCURD,
BlogCURD,
ArticleCommunity,
ArticleEmotion,
CitedArtiment,
Expand DownExpand Up@@ -102,6 +103,11 @@ defmodule GroupherServer.CMS do

defdelegate archive_articles(thread), to: ArticleCURD

defdelegate create_blog(community, attrs, user), to: BlogCURD
defdelegate create_blog_rss(attrs), to: BlogCURD
defdelegate update_blog_rss(attrs), to: BlogCURD
defdelegate blog_rss_info(rss), to: BlogCURD

defdelegate paged_citing_contents(type, id, filter), to: CitedArtiment

defdelegate upvote_article(thread, article_id, user), to: ArticleUpvote
Expand Down
137 changes: 137 additions & 0 deletionslib/groupher_server/cms/delegates/blog_curd.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
defmodule GroupherServer.CMS.Delegate.BlogCURD do
@moduledoc """
CURD operation on post/job ...
"""
import Ecto.Query, warn: false
import Helper.Utils, only: [strip_struct: 1, done: 1]
import Helper.ErrorCode

import GroupherServer.CMS.Delegate.ArticleCURD, only: [create_article: 4]
# import Helper.Utils, only: [done: 1]

# import Helper.ErrorCode
# import ShortMaps

# alias Helper.{ORM}
alias GroupherServer.{Accounts, CMS, Repo}
alias CMS.Model.{BlogRSS, Community}
alias Accounts.Model.User

alias Helper.{ORM, Cache, RSS}

@cache_pool :blog_rss

# alias Ecto.Multi
def blog_rss_info(rss) when is_binary(rss) do
with {:ok, feed} <- ORM.find_by(BlogRSS, %{rss: rss}) do
{:ok, feed}
else
_ -> fetch_fresh_rssinfo_and_cache(rss)
end
end

# attrs 包含 rss, blog_title
# def create_article(%Community{id: cid}, thread, attrs, %User{id: uid}) do
def create_blog(%Community{} = community, attrs, %User{} = user) do
# 1. 先判断 rss 是否存在
## 1.1 如果存在，从 cache 中获取
## 1.2 如不存在，则创建一条 RSS
with {:ok, feed} <- blog_rss_info(attrs.rss) do
do_create_blog(community, attrs, user, feed)

# IO.inspect(feed, label: "create blog")
# 通过 feed 有没有 id 来 insert / update
# 通过 blog_title, 组合 attrs 传给 create_article
end

# 2. 创建 blog
## 2.1 blog +字段 rss, author
## 2.2 title, digest, xxx

# 前台获取作者信息的时候从 rss 表读取
end

# rss 记录存在, 直接创建 blog
defp do_create_blog(%Community{} = community, attrs, %User{} = user, %{id: _} = feed) do
blog_author = if is_nil(feed.author), do: nil, else: Map.from_struct(feed.author)
selected_feed = Enum.find(feed.history_feed, &(&1.title == attrs.title))

# TODO: feed_digest, feed_content
attrs =
attrs
|> Map.merge(%{
link_addr: selected_feed.link_addr,
published: selected_feed.published,
blog_author: blog_author
})
|> Enum.reject(fn {_, v} -> is_nil(v) end)
|> Map.new()

create_article(community, :blog, attrs, user)
end

# rss 记录不存在, 先创建 rss, 再创建 blog
defp do_create_blog(%Community{} = community, attrs, %User{} = user, feed) do
with {:ok, feed} <- CMS.blog_rss_info(attrs.rss),
{:ok, feed} <- create_blog_rss(feed) do
do_create_blog(community, attrs, user, feed)
end
end

def create_blog_rss(attrs) do
history_feed = Map.get(attrs, :history_feed)
attrs = attrs |> Map.drop([:history_feed])

%BlogRSS{}
|> Ecto.Changeset.change(attrs)
|> Ecto.Changeset.put_embed(:history_feed, history_feed)
|> Repo.insert()
end

def update_blog_rss(%{rss: rss} = attrs) do
with {:ok, blog_rss} <- ORM.find_by(BlogRSS, rss: rss) do
history_feed =
Map.get(attrs, :history_feed, Enum.map(blog_rss.history_feed, &strip_struct(&1)))

attrs = attrs |> Map.drop([:history_feed])

%BlogRSS{}
|> Ecto.Changeset.change(attrs)
|> Ecto.Changeset.put_embed(:history_feed, history_feed)
|> Repo.insert()
end
end

# create done
# defp result({:ok, %{set_active_at_timestamp: result}}) do
# {:ok, result}
# end

# defp result({:ok, %{update_article_meta: result}}), do: {:ok, result}

# defp result({:error, :create_article, _result, _steps}) do
# {:error, [message: "create article", code: ecode(:create_fails)]}
# end

# defp result({:error, _, result, _steps}), do: {:error, result}

@doc """
get and cache feed by rss address as key
"""
def fetch_fresh_rssinfo_and_cache(rss) do
case Cache.get(@cache_pool, rss) do
{:ok, rssinfo} -> {:ok, rssinfo}
{:error, _} -> get_rssinfo_and_cache(rss)
end
end

defp get_rssinfo_and_cache(rss) do
# {:ok, feed} = RSS.get(rss)
with {:ok, rssinfo} <- RSS.get(rss) do
Cache.put(@cache_pool, rss, rssinfo)
{:ok, rssinfo}
else
{:error, _} -> {:error, [message: "blog rss is invalid", code: ecode(:invalid_blog_rss)]}
end
end
end
9 changes: 8 additions & 1 deletionlib/groupher_server/cms/models/blog.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,13 +15,18 @@ defmodule GroupherServer.CMS.Model.Blog do

@required_fields ~w(title digest)a
@article_cast_fields general_article_cast_fields()
@optional_fields ~w(digest)a ++ @article_cast_fields
@optional_fields ~w(digest feed_digest feed_content published)a ++ @article_cast_fields

@type t :: %Blog{}
schema "cms_blogs" do
# for frontend constant
field(:copy_right, :string, default: "", virtual: true)

field(:feed_digest, :string)
field(:feed_content, :string)
field(:published, :string)
embeds_one(:blog_author, Embeds.BlogAuthor, on_replace: :update)

article_tags_field(:blog)
article_communities_field(:blog)
general_article_fields(:blog)
Expand All@@ -33,13 +38,15 @@ defmodule GroupherServer.CMS.Model.Blog do
|> cast(attrs, @optional_fields ++ @required_fields)
|> validate_required(@required_fields)
|> cast_embed(:meta, required: false, with: &Embeds.ArticleMeta.changeset/2)
|> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2)
|> generl_changeset
end

@doc false
def update_changeset(%Blog{} = blog, attrs) do
blog
|> cast(attrs, @optional_fields ++ @required_fields)
|> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2)
|> generl_changeset
end

Expand Down
60 changes: 60 additions & 0 deletionslib/groupher_server/cms/models/blog_rss.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
defmodule GroupherServer.CMS.Model.BlogRSS do
@moduledoc false
alias __MODULE__

use Ecto.Schema
use Accessible

import Ecto.Changeset
# import GroupherServer.CMS.Helper.Macros

alias GroupherServer.CMS
alias CMS.Model.Embeds

@timestamps_opts [type: :utc_datetime_usec]

@required_fields ~w(link rss)a
@optional_fields ~w(subtitle author updated)a

@type t :: %BlogRSS{}
schema "cms_blog_rss" do
field(:rss, :string)
field(:title, :string)
field(:subtitle, :string)
field(:link, :string)
field(:updated, :string)
embeds_many(:history_feed, Embeds.BlogHistoryFeed, on_replace: :delete)
embeds_one(:author, Embeds.BlogAuthor, on_replace: :update)
end

@doc false
def changeset(%BlogRSS{} = blog_rss, attrs) do
blog_rss
|> cast(attrs, @optional_fields ++ @required_fields)
|> validate_required(@required_fields)
|> cast_embed(:history_feed, required: true, with: &Embeds.BlogHistoryFeed.changeset/2)
|> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2)
end

@doc false
def update_changeset(%BlogRSS{} = blog_rss, attrs) do
blog_rss
|> cast(attrs, @optional_fields ++ @required_fields)
|> cast_embed(:history_feed, required: false, with: &Embeds.BlogHistoryFeed.changeset/2)
|> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2)
end

# @doc false
# def update_changeset(%BlogRSS{} = blog_rss, attrs) do
# blog_rss
# |> cast(attrs, @optional_fields ++ @required_fields)
# |> generl_changeset
# end

# defp generl_changeset(changeset) do
# changeset
# |> validate_length(:title, min: 3, max: 100)
# |> cast_embed(:emotions, with: &Embeds.ArticleEmotion.changeset/2)
# |> validate_length(:link_addr, min: 5, max: 400)
# end
end
26 changes: 26 additions & 0 deletionslib/groupher_server/cms/models/embeds/blog_author.ex
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
defmodule GroupherServer.CMS.Model.Embeds.BlogAuthor do
@moduledoc """
general community meta
"""
use Ecto.Schema
use Accessible

import Ecto.Changeset

@required_fields ~w(name)a
@optional_fields ~w(link intro github twitter)a

embedded_schema do
field(:name, :string)
field(:link, :string)
field(:intro, :string)
field(:github, :string)
field(:twitter, :string)
end

def changeset(struct, attrs) do
struct
|> cast(attrs, @optional_fields ++ @required_fields)
|> validate_required(@required_fields)
end
end
Loading
[8]ページ先頭
©2009-2025 Movatter.jp