diff --git a/.app_version b/.app_version index 41915c79..61e6e92d 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.19.1 +0.19.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index b4cb6251..88b0a520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.19.2 - 2024-12-04 + +## The Telemetry release + +Dawarich now can collect usage metrics and send them to InfluxDB. Before this release, the only metrics that could be somehow tracked by developers (only @Freika, as of now) were the number of stars on GitHub and the overall number of docker images being pulled, across all versions of Dawarich, non-splittable by version. New in-app telemetry will allow us to track more granular metrics, allowing me to make decisions based on facts, not just guesses. + +I'm aware about the privacy concerns, so I want to be very transparent about what data is being sent and how it's used. + +Data being sent: + +- Number of DAU (Daily Active Users) +- App version +- Instance ID (unique identifier of the Dawarich instance built by hashing the api key of the first user in the database) + +The data is being sent to a InfluxDB instance hosted by me and won't be shared with anyone. + +Basically this set of metrics allows me to see how many people are using Dawarich and what versions they are using. No other data is being sent, nor it gives me any knowledge about individual users or their data or activity. + +The telemetry is enabled by default, but it **can be disabled** by setting `DISABLE_TELEMETRY` env var to `true`. The dataset might change in the future, but any changes will be documented here in the changelog and in every release as well as on the [telemetry page](https://dawarich.app/docs/tutorials/telemetry) of the website docs. + +### Added + +- Telemetry feature. It's now collecting usage metrics and sending them to InfluxDB. + # 0.19.1 - 2024-12-04 ### Fixed diff --git a/app/jobs/telemetry_sending_job.rb b/app/jobs/telemetry_sending_job.rb new file mode 100644 index 00000000..5b84f11a --- /dev/null +++ b/app/jobs/telemetry_sending_job.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class TelemetrySendingJob < ApplicationJob + queue_as :default + + def perform + return if ENV['DISABLE_TELEMETRY'] == 'true' + + data = Telemetry::Gather.new.call + Rails.logger.info("Telemetry data: #{data}") + + Telemetry::Send.new(data).call + end +end diff --git a/app/models/user.rb b/app/models/user.rb index 53adfa2d..58ce091d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -2,9 +2,9 @@ class User < ApplicationRecord # Include default devise modules. Others available are: - # :confirmable, :lockable, :timeoutable, :trackable and :omniauthable + # :confirmable, :lockable, :timeoutable, and :omniauthable devise :database_authenticatable, :registerable, - :recoverable, :rememberable, :validatable + :recoverable, :rememberable, :validatable, :trackable has_many :tracked_points, class_name: 'Point', dependent: :destroy has_many :imports, dependent: :destroy diff --git a/app/services/telemetry/gather.rb b/app/services/telemetry/gather.rb new file mode 100644 index 00000000..90b7ee01 --- /dev/null +++ b/app/services/telemetry/gather.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class Telemetry::Gather + def initialize(measurement: 'dawarich_usage_metrics') + @measurement = measurement + end + + def call + { + measurement:, + timestamp: Time.current.to_i, + tags: { instance_id: }, + fields: { dau:, app_version: } + } + end + + private + + attr_reader :measurement + + def instance_id + @instance_id ||= Digest::SHA2.hexdigest(User.first.api_key) + end + + def app_version + "\"#{APP_VERSION}\"" + end + + def dau + User.where(last_sign_in_at: Time.zone.today.beginning_of_day..Time.zone.today.end_of_day).count + end +end diff --git a/app/services/telemetry/send.rb b/app/services/telemetry/send.rb new file mode 100644 index 00000000..46401294 --- /dev/null +++ b/app/services/telemetry/send.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class Telemetry::Send + BUCKET = 'dawarich_metrics' + ORG = 'monitoring' + + def initialize(payload) + @payload = payload + end + + def call + return if ENV['DISABLE_TELEMETRY'] == 'true' + + line_protocol = build_line_protocol + response = send_request(line_protocol) + handle_response(response) + end + + private + + attr_reader :payload + + def build_line_protocol + tag_string = payload[:tags].map { |k, v| "#{k}=#{v}" }.join(',') + field_string = payload[:fields].map { |k, v| "#{k}=#{v}" }.join(',') + + "#{payload[:measurement]},#{tag_string} #{field_string} #{payload[:timestamp].to_i}" + end + + def send_request(line_protocol) + HTTParty.post( + "#{TELEMETRY_URL}?org=#{ORG}&bucket=#{BUCKET}&precision=s", + body: line_protocol, + headers: { + 'Authorization' => "Token #{Base64.decode64(TELEMETRY_STRING)}", + 'Content-Type' => 'text/plain' + } + ) + end + + def handle_response(response) + Rails.logger.error("InfluxDB write failed: #{response.body}") unless response.success? + + response + end +end diff --git a/config/initializers/01_constants.rb b/config/initializers/01_constants.rb index 5065345f..d8cc2d81 100644 --- a/config/initializers/01_constants.rb +++ b/config/initializers/01_constants.rb @@ -6,3 +6,5 @@ PHOTON_API_HOST = ENV.fetch('PHOTON_API_HOST', nil) PHOTON_API_USE_HTTPS = ENV.fetch('PHOTON_API_USE_HTTPS', 'true') == 'true' DISTANCE_UNIT = ENV.fetch('DISTANCE_UNIT', 'km').to_sym APP_VERSION = File.read('.app_version').strip +TELEMETRY_STRING = Base64.encode64('IjVFvb8j3P9-ArqhSGav9j8YcJaQiuNIzkfOPKQDk2lvKXqb8t1NSRv50oBkaKtlrB_ZRzO9NdurpMtncV_HYQ==') +TELEMETRY_URL = 'https://influxdb2.frey.today/api/v2/write' diff --git a/config/schedule.yml b/config/schedule.yml index 1b9a4f59..0b99f8c1 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -25,3 +25,8 @@ app_version_checking_job: cron: "0 */6 * * *" # every 6 hours class: "AppVersionCheckingJob" queue: default + +telemetry_sending_job: + cron: "0 */1 * * *" # every 1 hour + class: "TelemetrySendingJob" + queue: default diff --git a/db/migrate/20241205160055_add_devise_trackable_columns_to_users.rb b/db/migrate/20241205160055_add_devise_trackable_columns_to_users.rb new file mode 100644 index 00000000..80cccf4a --- /dev/null +++ b/db/migrate/20241205160055_add_devise_trackable_columns_to_users.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddDeviseTrackableColumnsToUsers < ActiveRecord::Migration[7.2] + def change + change_table :users, bulk: true do |t| + t.integer :sign_in_count, default: 0, null: false + t.datetime :current_sign_in_at + t.datetime :last_sign_in_at + t.string :current_sign_in_ip + t.string :last_sign_in_ip + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 3e1a538f..2927e2d5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do +ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -155,6 +155,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do t.bigint "user_id" t.jsonb "geodata", default: {}, null: false t.bigint "visit_id" + t.datetime "reverse_geocoded_at" t.index ["altitude"], name: "index_points_on_altitude" t.index ["battery"], name: "index_points_on_battery" t.index ["battery_status"], name: "index_points_on_battery_status" @@ -164,6 +165,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do t.index ["geodata"], name: "index_points_on_geodata", using: :gin t.index ["import_id"], name: "index_points_on_import_id" t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude" + t.index ["reverse_geocoded_at"], name: "index_points_on_reverse_geocoded_at" t.index ["timestamp"], name: "index_points_on_timestamp" t.index ["trigger"], name: "index_points_on_trigger" t.index ["user_id"], name: "index_points_on_user_id" @@ -208,6 +210,11 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do t.string "theme", default: "dark", null: false t.jsonb "settings", default: {"fog_of_war_meters"=>"100", "meters_between_routes"=>"1000", "minutes_between_routes"=>"60"} t.boolean "admin", default: false + t.integer "sign_in_count", default: 0, null: false + t.datetime "current_sign_in_at" + t.datetime "last_sign_in_at" + t.string "current_sign_in_ip" + t.string "last_sign_in_ip" t.index ["email"], name: "index_users_on_email", unique: true t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end diff --git a/spec/fixtures/files/geojson/export_same_points.json b/spec/fixtures/files/geojson/export_same_points.json index 2ecfb883..45fbe6a2 100644 --- a/spec/fixtures/files/geojson/export_same_points.json +++ b/spec/fixtures/files/geojson/export_same_points.json @@ -1 +1 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{}}}]} +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":null}}]} diff --git a/spec/jobs/telemetry_sending_job_spec.rb b/spec/jobs/telemetry_sending_job_spec.rb new file mode 100644 index 00000000..0acef0ee --- /dev/null +++ b/spec/jobs/telemetry_sending_job_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TelemetrySendingJob, type: :job do + describe '#perform' do + let(:gather_service) { instance_double(Telemetry::Gather) } + let(:send_service) { instance_double(Telemetry::Send) } + let(:telemetry_data) { { some: 'data' } } + + before do + allow(Telemetry::Gather).to receive(:new).and_return(gather_service) + allow(gather_service).to receive(:call).and_return(telemetry_data) + allow(Telemetry::Send).to receive(:new).with(telemetry_data).and_return(send_service) + allow(send_service).to receive(:call) + end + + it 'gathers telemetry data and sends it' do + described_class.perform_now + + expect(gather_service).to have_received(:call) + expect(send_service).to have_received(:call) + end + + context 'when DISABLE_TELEMETRY is set to true' do + before do + stub_const('ENV', ENV.to_h.merge('DISABLE_TELEMETRY' => 'true')) + end + + it 'does not send telemetry data' do + described_class.perform_now + + expect(send_service).not_to have_received(:call) + end + end + end +end diff --git a/spec/services/telemetry/gather_spec.rb b/spec/services/telemetry/gather_spec.rb new file mode 100644 index 00000000..9b962113 --- /dev/null +++ b/spec/services/telemetry/gather_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Telemetry::Gather do + let!(:user) { create(:user, last_sign_in_at: Time.zone.today) } + + describe '#call' do + subject(:gather) { described_class.new.call } + + it 'returns a hash with measurement, timestamp, tags, and fields' do + expect(gather).to include(:measurement, :timestamp, :tags, :fields) + end + + it 'includes the correct measurement' do + expect(gather[:measurement]).to eq('dawarich_usage_metrics') + end + + it 'includes the current timestamp' do + expect(gather[:timestamp]).to be_within(1).of(Time.current.to_i) + end + + it 'includes the correct instance_id in tags' do + expect(gather[:tags][:instance_id]).to eq(Digest::SHA2.hexdigest(user.api_key)) + end + + it 'includes the correct app_version in fields' do + expect(gather[:fields][:app_version]).to eq("\"#{APP_VERSION}\"") + end + + it 'includes the correct dau in fields' do + expect(gather[:fields][:dau]).to eq(1) + end + + context 'with a custom measurement' do + let(:measurement) { 'custom_measurement' } + + subject(:gather) { described_class.new(measurement:).call } + + it 'includes the correct measurement' do + expect(gather[:measurement]).to eq('custom_measurement') + end + end + end +end