From 9bd23dc4e51ba47283a8e3a66cd94b4e624a5235 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 27 May 2018 21:45:30 +0200 Subject: [PATCH] Track trending tags (#7638) * Track trending tags - Half-life of 1 day - Historical usage in daily buckets (last 7 days stored) - GET /api/v1/trends Fix #271 * Add trends to web UI * Don't render compose form on search route, adjust search results header * Disqualify tag from trends if it's in disallowed hashtags setting * Count distinct accounts using tag, ignore silenced accounts --- app/controllers/api/v1/trends_controller.rb | 17 ++++ app/javascript/mastodon/actions/trends.js | 32 +++++++ .../compose/components/search_results.js | 59 ++++++++++++- .../containers/search_results_container.js | 8 +- .../mastodon/features/compose/index.js | 4 +- app/javascript/mastodon/reducers/index.js | 2 + app/javascript/mastodon/reducers/trends.js | 13 +++ .../styles/mastodon/components.scss | 83 ++++++++++++++++++- app/models/tag.rb | 16 ++++ app/models/trending_tags.rb | 61 ++++++++++++++ app/serializers/rest/tag_serializer.rb | 11 +++ app/services/process_hashtags_service.rb | 6 +- config/routes.rb | 1 + package.json | 1 + yarn.lock | 6 ++ 15 files changed, 310 insertions(+), 10 deletions(-) create mode 100644 app/controllers/api/v1/trends_controller.rb create mode 100644 app/javascript/mastodon/actions/trends.js create mode 100644 app/javascript/mastodon/reducers/trends.js create mode 100644 app/models/trending_tags.rb create mode 100644 app/serializers/rest/tag_serializer.rb diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb new file mode 100644 index 000000000..bcea9857e --- /dev/null +++ b/app/controllers/api/v1/trends_controller.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class Api::V1::TrendsController < Api::BaseController + before_action :set_tags + + respond_to :json + + def index + render json: @tags, each_serializer: REST::TagSerializer + end + + private + + def set_tags + @tags = TrendingTags.get(limit_param(10)) + end +end diff --git a/app/javascript/mastodon/actions/trends.js b/app/javascript/mastodon/actions/trends.js new file mode 100644 index 000000000..853e4f60a --- /dev/null +++ b/app/javascript/mastodon/actions/trends.js @@ -0,0 +1,32 @@ +import api from '../api'; + +export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST'; +export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS'; +export const TRENDS_FETCH_FAIL = 'TRENDS_FETCH_FAIL'; + +export const fetchTrends = () => (dispatch, getState) => { + dispatch(fetchTrendsRequest()); + + api(getState) + .get('/api/v1/trends') + .then(({ data }) => dispatch(fetchTrendsSuccess(data))) + .catch(err => dispatch(fetchTrendsFail(err))); +}; + +export const fetchTrendsRequest = () => ({ + type: TRENDS_FETCH_REQUEST, + skipLoading: true, +}); + +export const fetchTrendsSuccess = trends => ({ + type: TRENDS_FETCH_SUCCESS, + trends, + skipLoading: true, +}); + +export const fetchTrendsFail = error => ({ + type: TRENDS_FETCH_FAIL, + error, + skipLoading: true, + skipAlert: true, +}); diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js index 84455563c..f2655c14d 100644 --- a/app/javascript/mastodon/features/compose/components/search_results.js +++ b/app/javascript/mastodon/features/compose/components/search_results.js @@ -1,23 +1,75 @@ import React from 'react'; +import PropTypes from 'prop-types'; import ImmutablePropTypes from 'react-immutable-proptypes'; -import { FormattedMessage } from 'react-intl'; +import { FormattedMessage, FormattedNumber } from 'react-intl'; import AccountContainer from '../../../containers/account_container'; import StatusContainer from '../../../containers/status_container'; import { Link } from 'react-router-dom'; import ImmutablePureComponent from 'react-immutable-pure-component'; +import { Sparklines, SparklinesCurve } from 'react-sparklines'; + +const shortNumberFormat = number => { + if (number < 1000) { + return ; + } else { + return K; + } +}; export default class SearchResults extends ImmutablePureComponent { static propTypes = { results: ImmutablePropTypes.map.isRequired, + trends: ImmutablePropTypes.list, + fetchTrends: PropTypes.func.isRequired, }; + componentDidMount () { + const { fetchTrends } = this.props; + fetchTrends(); + } + render () { - const { results } = this.props; + const { results, trends } = this.props; let accounts, statuses, hashtags; let count = 0; + if (results.isEmpty()) { + return ( +
+
+
+ + +
+ + {trends && trends.map(hashtag => ( +
+
+ + #{hashtag.get('name')} + + + {shortNumberFormat(hashtag.getIn(['history', 0, 'accounts']))} }} /> +
+ +
+ {shortNumberFormat(hashtag.getIn(['history', 0, 'uses']))} +
+ +
+ day.get('uses')).toArray()}> + + +
+
+ ))} +
+
+ ); + } + if (results.get('accounts') && results.get('accounts').size > 0) { count += results.get('accounts').size; accounts = ( @@ -48,7 +100,7 @@ export default class SearchResults extends ImmutablePureComponent { {results.get('hashtags').map(hashtag => ( - #{hashtag} + {hashtag} ))} @@ -58,6 +110,7 @@ export default class SearchResults extends ImmutablePureComponent { return (
+
diff --git a/app/javascript/mastodon/features/compose/containers/search_results_container.js b/app/javascript/mastodon/features/compose/containers/search_results_container.js index 16d95d417..7273460e2 100644 --- a/app/javascript/mastodon/features/compose/containers/search_results_container.js +++ b/app/javascript/mastodon/features/compose/containers/search_results_container.js @@ -1,8 +1,14 @@ import { connect } from 'react-redux'; import SearchResults from '../components/search_results'; +import { fetchTrends } from '../../../actions/trends'; const mapStateToProps = state => ({ results: state.getIn(['search', 'results']), + trends: state.get('trends'), }); -export default connect(mapStateToProps)(SearchResults); +const mapDispatchToProps = dispatch => ({ + fetchTrends: () => dispatch(fetchTrends()), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(SearchResults); diff --git a/app/javascript/mastodon/features/compose/index.js b/app/javascript/mastodon/features/compose/index.js index 19aae0332..d8e9ad9ee 100644 --- a/app/javascript/mastodon/features/compose/index.js +++ b/app/javascript/mastodon/features/compose/index.js @@ -101,7 +101,7 @@ export default class Compose extends React.PureComponent { {(multiColumn || isSearchPage) && }
-
+ {!isSearchPage &&
{multiColumn && ( @@ -109,7 +109,7 @@ export default class Compose extends React.PureComponent {
)} -
+
} {({ x }) => ( diff --git a/app/javascript/mastodon/reducers/index.js b/app/javascript/mastodon/reducers/index.js index 3d9a6a132..019c1f466 100644 --- a/app/javascript/mastodon/reducers/index.js +++ b/app/javascript/mastodon/reducers/index.js @@ -26,6 +26,7 @@ import height_cache from './height_cache'; import custom_emojis from './custom_emojis'; import lists from './lists'; import listEditor from './list_editor'; +import trends from './trends'; const reducers = { dropdown_menu, @@ -55,6 +56,7 @@ const reducers = { custom_emojis, lists, listEditor, + trends, }; export default combineReducers(reducers); diff --git a/app/javascript/mastodon/reducers/trends.js b/app/javascript/mastodon/reducers/trends.js new file mode 100644 index 000000000..95cf8f284 --- /dev/null +++ b/app/javascript/mastodon/reducers/trends.js @@ -0,0 +1,13 @@ +import { TRENDS_FETCH_SUCCESS } from '../actions/trends'; +import { fromJS } from 'immutable'; + +const initialState = null; + +export default function trendsReducer(state = initialState, action) { + switch(action.type) { + case TRENDS_FETCH_SUCCESS: + return fromJS(action.trends); + default: + return state; + } +}; diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 2724454fb..c66bc427c 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -3334,9 +3334,15 @@ a.status-card { color: $dark-text-color; background: lighten($ui-base-color, 2%); border-bottom: 1px solid darken($ui-base-color, 4%); - padding: 15px 10px; - font-size: 14px; + padding: 15px; font-weight: 500; + font-size: 16px; + cursor: default; + + .fa { + display: inline-block; + margin-right: 5px; + } } .search-results__section { @@ -5209,3 +5215,76 @@ noscript { background: $ui-base-color; } } + +.trends { + &__header { + color: $dark-text-color; + background: lighten($ui-base-color, 2%); + border-bottom: 1px solid darken($ui-base-color, 4%); + font-weight: 500; + padding: 15px; + font-size: 16px; + cursor: default; + + .fa { + display: inline-block; + margin-right: 5px; + } + } + + &__item { + display: flex; + align-items: center; + padding: 15px; + border-bottom: 1px solid lighten($ui-base-color, 8%); + + &:last-child { + border-bottom: 0; + } + + &__name { + flex: 1 1 auto; + color: $dark-text-color; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + + strong { + font-weight: 500; + } + + a { + color: $darker-text-color; + text-decoration: none; + font-size: 14px; + font-weight: 500; + display: block; + + &:hover, + &:focus, + &:active { + span { + text-decoration: underline; + } + } + } + } + + &__current { + width: 100px; + font-size: 24px; + line-height: 36px; + font-weight: 500; + text-align: center; + color: $secondary-text-color; + } + + &__sparkline { + width: 50px; + + path { + stroke: lighten($highlight-text-color, 6%) !important; + } + } + } +} diff --git a/app/models/tag.rb b/app/models/tag.rb index 8b1b02412..4f31f796e 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -21,6 +21,22 @@ class Tag < ApplicationRecord name end + def history + days = [] + + 7.times do |i| + day = i.days.ago.beginning_of_day.to_i + + days << { + day: day.to_s, + uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', + accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, + } + end + + days + end + class << self def search_for(term, limit = 5) pattern = sanitize_sql_like(term.strip) + '%' diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb new file mode 100644 index 000000000..eedd92644 --- /dev/null +++ b/app/models/trending_tags.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class TrendingTags + KEY = 'trending_tags' + HALF_LIFE = 1.day.to_i + MAX_ITEMS = 500 + EXPIRE_HISTORY_AFTER = 7.days.seconds + + class << self + def record_use!(tag, account, at_time = Time.now.utc) + return if disallowed_hashtags.include?(tag.name) || account.silenced? + + increment_vote!(tag.id, at_time) + increment_historical_use!(tag.id, at_time) + increment_unique_use!(tag.id, account.id, at_time) + end + + def get(limit) + tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i) + tags = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h + tag_ids.map { |tag_id| tags[tag_id] }.compact + end + + private + + def increment_vote!(tag_id, at_time) + redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s) + redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS) + end + + def increment_historical_use!(tag_id, at_time) + key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" + redis.incrby(key, 1) + redis.expire(key, EXPIRE_HISTORY_AFTER) + end + + def increment_unique_use!(tag_id, account_id, at_time) + key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" + redis.pfadd(key, account_id) + redis.expire(key, EXPIRE_HISTORY_AFTER) + end + + # The epoch needs to be 2.5 years in the future if the half-life is one day + # While dynamic, it will always be the same within one year + def epoch + @epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i + end + + def disallowed_hashtags + return @disallowed_hashtags if defined?(@disallowed_hashtags) + + @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags + @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String + @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) + end + + def redis + Redis.current + end + end +end diff --git a/app/serializers/rest/tag_serializer.rb b/app/serializers/rest/tag_serializer.rb new file mode 100644 index 000000000..74aa571a4 --- /dev/null +++ b/app/serializers/rest/tag_serializer.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class REST::TagSerializer < ActiveModel::Serializer + include RoutingHelper + + attributes :name, :url, :history + + def url + tag_url(object) + end +end diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index 5b45c865f..0695922b8 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -4,8 +4,10 @@ class ProcessHashtagsService < BaseService def call(status, tags = []) tags = Extractor.extract_hashtags(status.text) if status.local? - tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag| - status.tags << Tag.where(name: tag).first_or_initialize(name: tag) + tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name| + tag = Tag.where(name: name).first_or_create(name: name) + status.tags << tag + TrendingTags.record_use!(tag, status.account, status.created_at) end end end diff --git a/config/routes.rb b/config/routes.rb index 3042b5ea0..2fcb885ed 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -254,6 +254,7 @@ Rails.application.routes.draw do resources :mutes, only: [:index] resources :favourites, only: [:index] resources :reports, only: [:index, :create] + resources :trends, only: [:index] namespace :apps do get :verify_credentials, to: 'credentials#show' diff --git a/package.json b/package.json index 61f38409c..6ee6f98d3 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "react-redux-loading-bar": "^2.9.3", "react-router-dom": "^4.1.1", "react-router-scroll-4": "^1.0.0-beta.1", + "react-sparklines": "^1.7.0", "react-swipeable-views": "^0.12.3", "react-textarea-autosize": "^5.2.1", "react-toggle": "^4.0.1", diff --git a/yarn.lock b/yarn.lock index 50c88557d..de48c995a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6124,6 +6124,12 @@ react-router@^4.2.0: prop-types "^15.5.4" warning "^3.0.0" +react-sparklines@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/react-sparklines/-/react-sparklines-1.7.0.tgz#9b1d97e8c8610095eeb2ad658d2e1fcf91f91a60" + dependencies: + prop-types "^15.5.10" + react-swipeable-views-core@^0.12.11: version "0.12.11" resolved "https://registry.yarnpkg.com/react-swipeable-views-core/-/react-swipeable-views-core-0.12.11.tgz#3cf2b4daffbb36f9d69bd19bf5b2d5370b6b2c1b"