Merge pull request #488 from Freika/feature/telemetry

Telemetry

.app_version

@@ -1 +1 @@
-0.19.1
+0.19.2

CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# 0.19.2 - 2024-12-04
+
+## The Telemetry release
+
+Dawarich now can collect usage metrics and send them to InfluxDB. Before this release, the only metrics that could be somehow tracked by developers (only @Freika, as of now) were the number of stars on GitHub and the overall number of docker images being pulled, across all versions of Dawarich, non-splittable by version. New in-app telemetry will allow us to track more granular metrics, allowing me to make decisions based on facts, not just guesses.
+
+I'm aware about the privacy concerns, so I want to be very transparent about what data is being sent and how it's used.
+
+Data being sent:
+
+- Number of DAU (Daily Active Users)
+- App version
+- Instance ID (unique identifier of the Dawarich instance built by hashing the api key of the first user in the database)
+
+The data is being sent to a InfluxDB instance hosted by me and won't be shared with anyone.
+
+Basically this set of metrics allows me to see how many people are using Dawarich and what versions they are using. No other data is being sent, nor it gives me any knowledge about individual users or their data or activity.
+
+The telemetry is enabled by default, but it **can be disabled** by setting `DISABLE_TELEMETRY` env var to `true`. The dataset might change in the future, but any changes will be documented here in the changelog and in every release as well as on the [telemetry page](https://dawarich.app/docs/tutorials/telemetry) of the website docs.
+
+### Added
+
+- Telemetry feature. It's now collecting usage metrics and sending them to InfluxDB.
+
 # 0.19.1 - 2024-12-04
 
 ### Fixed

app/jobs/telemetry_sending_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class TelemetrySendingJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    return if ENV['DISABLE_TELEMETRY'] == 'true'
+
+    data = Telemetry::Gather.new.call
+    Rails.logger.info("Telemetry data: #{data}")
+
+    Telemetry::Send.new(data).call
+  end
+end

app/models/user.rb

@@ -2,9 +2,9 @@
 
 class User < ApplicationRecord
   # Include default devise modules. Others available are:
-  # :confirmable, :lockable, :timeoutable, :trackable and :omniauthable
+  # :confirmable, :lockable, :timeoutable, and :omniauthable
   devise :database_authenticatable, :registerable,
-         :recoverable, :rememberable, :validatable
+         :recoverable, :rememberable, :validatable, :trackable
 
   has_many :tracked_points, class_name: 'Point', dependent: :destroy
   has_many :imports,        dependent: :destroy

app/services/telemetry/gather.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Telemetry::Gather
+  def initialize(measurement: 'dawarich_usage_metrics')
+    @measurement = measurement
+  end
+
+  def call
+    {
+      measurement:,
+      timestamp: Time.current.to_i,
+      tags: { instance_id: },
+      fields: { dau:, app_version: }
+    }
+  end
+
+  private
+
+  attr_reader :measurement
+
+  def instance_id
+    @instance_id ||= Digest::SHA2.hexdigest(User.first.api_key)
+  end
+
+  def app_version
+    "\"#{APP_VERSION}\""
+  end
+
+  def dau
+    User.where(last_sign_in_at: Time.zone.today.beginning_of_day..Time.zone.today.end_of_day).count
+  end
+end

app/services/telemetry/send.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+class Telemetry::Send
+  BUCKET = 'dawarich_metrics'
+  ORG = 'monitoring'
+
+  def initialize(payload)
+    @payload = payload
+  end
+
+  def call
+    return if ENV['DISABLE_TELEMETRY'] == 'true'
+
+    line_protocol = build_line_protocol
+    response = send_request(line_protocol)
+    handle_response(response)
+  end
+
+  private
+
+  attr_reader :payload
+
+  def build_line_protocol
+    tag_string = payload[:tags].map { |k, v| "#{k}=#{v}" }.join(',')
+    field_string = payload[:fields].map { |k, v| "#{k}=#{v}" }.join(',')
+
+    "#{payload[:measurement]},#{tag_string} #{field_string} #{payload[:timestamp].to_i}"
+  end
+
+  def send_request(line_protocol)
+    HTTParty.post(
+      "#{TELEMETRY_URL}?org=#{ORG}&bucket=#{BUCKET}&precision=s",
+      body: line_protocol,
+      headers: {
+        'Authorization' => "Token #{Base64.decode64(TELEMETRY_STRING)}",
+        'Content-Type' => 'text/plain'
+      }
+    )
+  end
+
+  def handle_response(response)
+    Rails.logger.error("InfluxDB write failed: #{response.body}") unless response.success?
+
+    response
+  end
+end

config/initializers/01_constants.rb

@@ -6,3 +6,5 @@ PHOTON_API_HOST = ENV.fetch('PHOTON_API_HOST', nil)
 PHOTON_API_USE_HTTPS = ENV.fetch('PHOTON_API_USE_HTTPS', 'true') == 'true'
 DISTANCE_UNIT = ENV.fetch('DISTANCE_UNIT', 'km').to_sym
 APP_VERSION = File.read('.app_version').strip
+TELEMETRY_STRING = Base64.encode64('IjVFvb8j3P9-ArqhSGav9j8YcJaQiuNIzkfOPKQDk2lvKXqb8t1NSRv50oBkaKtlrB_ZRzO9NdurpMtncV_HYQ==')
+TELEMETRY_URL = 'https://influxdb2.frey.today/api/v2/write'

config/schedule.yml

@@ -25,3 +25,8 @@ app_version_checking_job:
   cron: "0 */6 * * *" # every 6 hours
   class: "AppVersionCheckingJob"
   queue: default
+
+telemetry_sending_job:
+  cron: "0 */1 * * *" # every 1 hour
+  class: "TelemetrySendingJob"
+  queue: default

db/migrate/20241205160055_add_devise_trackable_columns_to_users.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddDeviseTrackableColumnsToUsers < ActiveRecord::Migration[7.2]
+  def change
+    change_table :users, bulk: true do |t|
+      t.integer :sign_in_count, default: 0, null: false
+      t.datetime :current_sign_in_at
+      t.datetime :last_sign_in_at
+      t.string :current_sign_in_ip
+      t.string :last_sign_in_ip
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do
+ActiveRecord::Schema[7.2].define(version: 2024_12_05_160055) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -155,6 +155,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do
     t.bigint "user_id"
     t.jsonb "geodata", default: {}, null: false
     t.bigint "visit_id"
+    t.datetime "reverse_geocoded_at"
     t.index ["altitude"], name: "index_points_on_altitude"
     t.index ["battery"], name: "index_points_on_battery"
     t.index ["battery_status"], name: "index_points_on_battery_status"
@@ -164,6 +165,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do
     t.index ["geodata"], name: "index_points_on_geodata", using: :gin
     t.index ["import_id"], name: "index_points_on_import_id"
     t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude"
+    t.index ["reverse_geocoded_at"], name: "index_points_on_reverse_geocoded_at"
     t.index ["timestamp"], name: "index_points_on_timestamp"
     t.index ["trigger"], name: "index_points_on_trigger"
     t.index ["user_id"], name: "index_points_on_user_id"
@@ -208,6 +210,11 @@ ActiveRecord::Schema[7.2].define(version: 2024_11_28_095325) do
     t.string "theme", default: "dark", null: false
     t.jsonb "settings", default: {"fog_of_war_meters"=>"100", "meters_between_routes"=>"1000", "minutes_between_routes"=>"60"}
     t.boolean "admin", default: false
+    t.integer "sign_in_count", default: 0, null: false
+    t.datetime "current_sign_in_at"
+    t.datetime "last_sign_in_at"
+    t.string "current_sign_in_ip"
+    t.string "last_sign_in_ip"
     t.index ["email"], name: "index_users_on_email", unique: true
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
   end

spec/jobs/telemetry_sending_job_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe TelemetrySendingJob, type: :job do
+  describe '#perform' do
+    let(:gather_service) { instance_double(Telemetry::Gather) }
+    let(:send_service) { instance_double(Telemetry::Send) }
+    let(:telemetry_data) { { some: 'data' } }
+
+    before do
+      allow(Telemetry::Gather).to receive(:new).and_return(gather_service)
+      allow(gather_service).to receive(:call).and_return(telemetry_data)
+      allow(Telemetry::Send).to receive(:new).with(telemetry_data).and_return(send_service)
+      allow(send_service).to receive(:call)
+    end
+
+    it 'gathers telemetry data and sends it' do
+      described_class.perform_now
+
+      expect(gather_service).to have_received(:call)
+      expect(send_service).to have_received(:call)
+    end
+
+    context 'when DISABLE_TELEMETRY is set to true' do
+      before do
+        stub_const('ENV', ENV.to_h.merge('DISABLE_TELEMETRY' => 'true'))
+      end
+
+      it 'does not send telemetry data' do
+        described_class.perform_now
+
+        expect(send_service).not_to have_received(:call)
+      end
+    end
+  end
+end

spec/services/telemetry/gather_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Telemetry::Gather do
+  let!(:user) { create(:user, last_sign_in_at: Time.zone.today) }
+
+  describe '#call' do
+    subject(:gather) { described_class.new.call }
+
+    it 'returns a hash with measurement, timestamp, tags, and fields' do
+      expect(gather).to include(:measurement, :timestamp, :tags, :fields)
+    end
+
+    it 'includes the correct measurement' do
+      expect(gather[:measurement]).to eq('dawarich_usage_metrics')
+    end
+
+    it 'includes the current timestamp' do
+      expect(gather[:timestamp]).to be_within(1).of(Time.current.to_i)
+    end
+
+    it 'includes the correct instance_id in tags' do
+      expect(gather[:tags][:instance_id]).to eq(Digest::SHA2.hexdigest(user.api_key))
+    end
+
+    it 'includes the correct app_version in fields' do
+      expect(gather[:fields][:app_version]).to eq("\"#{APP_VERSION}\"")
+    end
+
+    it 'includes the correct dau in fields' do
+      expect(gather[:fields][:dau]).to eq(1)
+    end
+
+    context 'with a custom measurement' do
+      let(:measurement) { 'custom_measurement' }
+
+      subject(:gather) { described_class.new(measurement:).call }
+
+      it 'includes the correct measurement' do
+        expect(gather[:measurement]).to eq('custom_measurement')
+      end
+    end
+  end
+end