diff --git a/.env.development b/.env.development index d01eb0be..f7f46a10 100644 --- a/.env.development +++ b/.env.development @@ -1,4 +1,4 @@ -DATABASE_HOST=dawarich_db +DATABASE_HOST=localhost DATABASE_USERNAME=postgres DATABASE_PASSWORD=password DATABASE_NAME=dawarich_development diff --git a/.env.template b/.env.template new file mode 100644 index 00000000..09753f71 --- /dev/null +++ b/.env.template @@ -0,0 +1 @@ +MAPBOX_ACCESS_TOKEN=MAPBOX_ACCESS_TOKEN diff --git a/Gemfile b/Gemfile index 7420b48d..8911ce6c 100644 --- a/Gemfile +++ b/Gemfile @@ -17,12 +17,17 @@ gem 'tailwindcss-rails' gem 'turbo-rails' gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] gem "importmap-rails" +gem "mapkick-rb" +gem 'geocoder' +gem 'sidekiq' + group :development, :test do gem 'debug', platforms: %i[mri mingw x64_mingw] gem 'factory_bot_rails' gem 'ffaker' gem 'rspec-rails' + gem 'dotenv-rails' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 96973f98..54684744 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -97,6 +97,10 @@ GEM warden (~> 1.2.3) diff-lcs (1.5.1) docile (1.4.0) + dotenv (3.1.0) + dotenv-rails (3.1.0) + dotenv (= 3.1.0) + railties (>= 6.1) drb (2.2.1) erubi (1.12.0) factory_bot (6.4.6) @@ -106,6 +110,7 @@ GEM railties (>= 5.0.0) ffaker (2.23.0) foreman (0.87.2) + geocoder (1.8.2) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.4) @@ -128,6 +133,7 @@ GEM net-imap net-pop net-smtp + mapkick-rb (0.1.5) marcel (1.0.4) mini_mime (1.1.5) minitest (5.22.3) @@ -252,6 +258,11 @@ GEM ruby-progressbar (1.13.0) shoulda-matchers (6.1.0) activesupport (>= 5.2.0) + sidekiq (7.2.2) + concurrent-ruby (< 2) + connection_pool (>= 2.3.0) + rack (>= 2.2.4) + redis-client (>= 0.19.0) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -305,10 +316,13 @@ DEPENDENCIES bootsnap debug devise + dotenv-rails factory_bot_rails ffaker foreman + geocoder importmap-rails + mapkick-rb pg puma pundit @@ -317,6 +331,7 @@ DEPENDENCIES rspec-rails rubocop-rails shoulda-matchers + sidekiq simplecov sprockets-rails stimulus-rails diff --git a/README.md b/README.md index 7159e644..444a4fc2 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ # Dawarich -This is a Rails 7.0.2.3 app template with test suite, user auth and development docker env. +This is a Rails app that receives location updates from Owntracks and stores them in a database. It also provides a web interface to view the location history. -## How to rename the app +## Features -Run +### Google Maps Timeline import -```bash -ruby rename_app.rb old_app_name new_app_name -``` +You can import your Google Maps Timeline data into Wardu. -Notice, the name must be in snake_case. Default app name is `dawarich`. +### Location history +You can view your location history on a map. ## How to start the app locally @@ -40,7 +39,7 @@ Dockerized with https://betterprogramming.pub/rails-6-development-with-docker-55 ```json { "scripts": { - "predeploy": "dokku ps:stop dawarich" + "predeploy": "dokku ps:stop wardu" }, "formation": { "web": { @@ -52,3 +51,102 @@ Dockerized with https://betterprogramming.pub/rails-6-development-with-docker-55 } } ``` + + +{ + "cog": 271, + "batt": 41, + "lon": 2.29513, + "acc": 5, + "vel": 61, + "vac": 21, + "lat": 48.85833, + "t": "u", + "tst": 1497508651, + "alt": 167, + "_type": "location", + "topic": "owntracks/jane/iphone", + "p": 71, + "tid": "JJ" +} + +{"bs"=>1, # battery status +"p"=>102.818, # ping +"batt"=>100, # battery +"_type"=>"location", # type +"tid"=>"RO", # Tracker ID used to display the initials of a user (iOS,Android/string/optional) required for http mode +"topic"=>"owntracks/Frey/iPhone 12 Pro", +"alt"=>36, +"lon"=>13.504178, +"vel"=>0, # velocity +"t"=>"u", +"BSSID"=>"b0:f2:8:45:94:33", +"SSID"=>"FRITZ!Box 6660 Cable LQ", +"conn"=>"w", # connection, w = wifi, m = mobile, o = offline +"vac"=>3, # vertical accuracy +"acc"=>5, # horizontal accuracy +"tst"=>1702662679, Timestamp at which the beacon was seen (iOS/integer/epoch) +"lat"=>52.445526, +"m"=>1, # mode, significant = 1, move = 2 +"inrids"=>["5f1d1b"], # contains a list of region IDs the device is currently in (e.g. ["6da9cf","3defa7"]). Might be empty. (iOS,Android/list of strings/optional) +"inregions"=>["home"], +"point"=>{"bs"=>1, +"p"=>102.818, +"batt"=>100, +"_type"=>"location", +"tid"=>"RO", +"topic"=>"owntracks/Frey/iPhone 12 Pro", +"alt"=>36, +"lon"=>13.504178, +"vel"=>0, +"t"=>"u", +"BSSID"=>"b0:f2:8:45:94:33", +"SSID"=>"FRITZ!Box 6660 Cable LQ", +"conn"=>"w", +"vac"=>3, +"acc"=>5, +"tst"=>1702662679, +"lat"=>52.445526, +"m"=>1, +"inrids"=>["5f1d1b"], +"inregions"=>["home"]}} +18:51:18 web.1 | #1, +"p"=>102.818, +"batt"=>100, +"_type"=>"location", +"tid"=>"RO", +"topic"=>"owntracks/Frey/iPhone 12 Pro", +"alt"=>36, +"lon"=>13.504178, +"vel"=>0, +"t"=>"u", +"BSSID"=>"b0:f2:8:45:94:33", +"SSID"=>"FRITZ!Box 6660 Cable LQ", +"conn"=>"w", +"vac"=>3, +"acc"=>5, +"tst"=>1702662679, +"lat"=>52.445526, +"m"=>1, +"inrids"=>["5f1d1b"], +"inregions"=>["home"], +"point"=>{"bs"=>1, +"p"=>102.818, +"batt"=>100, +"_type"=>"location", +"tid"=>"RO", +"topic"=>"owntracks/Frey/iPhone 12 Pro", +"alt"=>36, +"lon"=>13.504178, +"vel"=>0, +"t"=>"u", +"BSSID"=>"b0:f2:8:45:94:33", +"SSID"=>"FRITZ!Box 6660 Cable LQ", +"conn"=>"w", +"vac"=>3, +"acc"=>5, +"tst"=>1702662679, +"lat"=>52.445526, +"m"=>1, +"inrids"=>["5f1d1b"], +"inregions"=>["home"]}} permitted: false> diff --git a/app/controllers/api/v1/points_controller.rb b/app/controllers/api/v1/points_controller.rb new file mode 100644 index 00000000..1a1e8b8f --- /dev/null +++ b/app/controllers/api/v1/points_controller.rb @@ -0,0 +1,24 @@ +class Api::V1::PointsController < ApplicationController + skip_forgery_protection + + def create + parsed_params = OwnTracks::Params.new(point_params).call + + @point = Point.create(parsed_params) + + if @point.valid? + render json: @point, status: :ok + else + render json: @point.errors, status: :unprocessable_entity + end + end + + private + + def point_params + params.permit( + :lat, :lon, :bs, :batt, :p, :alt, :acc, :vac, :vel, :conn, :SSID, :BSSID, :m, :tid, :tst, + :topic, :_type, :cog, :t, inrids: [], inregions: [], point: {} + ) + end +end diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index 6afda6b6..79ae91e2 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -1,7 +1,9 @@ class HomeController < ApplicationController def index - # if current_user - # redirect_to dashboard_path - # end + if current_user + redirect_to points_url + end + + @points = current_user.points if current_user end end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb new file mode 100644 index 00000000..184e7ff6 --- /dev/null +++ b/app/controllers/imports_controller.rb @@ -0,0 +1,60 @@ +class ImportsController < ApplicationController + before_action :authenticate_user! + before_action :set_import, only: %i[ show destroy ] + + def index + @imports = current_user.imports + end + + def show + end + + def new + @import = Import.new + end + + def create + files = import_params[:files].reject(&:blank?) + imports = [] + success = true + + files.each do |file| + json = JSON.parse(file.read) + import = current_user.imports.create(name: file.original_filename) + parser.new(file.path, import.id).call + + imports << import + end + + redirect_to imports_url, notice: "#{imports.count} imports was successfully created.", status: :see_other + + rescue StandardError => e + imports.each { |import| import&.destroy! } + + flash.now[:error] = e.message + + redirect_to new_import_path, notice: e.message, status: :unprocessable_entity + end + + def destroy + @import.destroy! + redirect_to imports_url, notice: "Import was successfully destroyed.", status: :see_other + end + + private + + def set_import + @import = Import.find(params[:id]) + end + + def import_params + params.require(:import).permit(:source, files: []) + end + + def parser + case params[:import][:source] + when 'google' then GoogleMaps::TimelineParser + when 'owntracks' then OwnTracks::ExportParser + end + end +end diff --git a/app/controllers/points_controller.rb b/app/controllers/points_controller.rb new file mode 100644 index 00000000..298eb813 --- /dev/null +++ b/app/controllers/points_controller.rb @@ -0,0 +1,7 @@ +class PointsController < ApplicationController + def index + @points = Point.all + + @coordinates = @points.as_json(only: [:latitude, :longitude]) + end +end diff --git a/app/helpers/points_helper.rb b/app/helpers/points_helper.rb new file mode 100644 index 00000000..07e88a84 --- /dev/null +++ b/app/helpers/points_helper.rb @@ -0,0 +1,2 @@ +module PointsHelper +end diff --git a/app/javascript/application.js b/app/javascript/application.js index caea8389..8b283c93 100644 --- a/app/javascript/application.js +++ b/app/javascript/application.js @@ -3,3 +3,7 @@ import "@rails/actioncable" import "controllers" import "@hotwired/turbo-rails" + +import "mapkick/bundle" +import "leaflet" +import "leaflet-providers" diff --git a/app/jobs/reverse_geocoding_job.rb b/app/jobs/reverse_geocoding_job.rb new file mode 100644 index 00000000..ae556cd3 --- /dev/null +++ b/app/jobs/reverse_geocoding_job.rb @@ -0,0 +1,16 @@ +class ReverseGeocodingJob < ApplicationJob + queue_as :default + + def perform(point_id) + point = Point.find(point_id) + return if point.city.present? && point.country.present? + + result = Geocoder.search([point.latitude, point.longitude]) + return if result.blank? + + point.update( + city: result.first.city, + country: result.first.country + ) + end +end diff --git a/app/models/import.rb b/app/models/import.rb new file mode 100644 index 00000000..02f1db7e --- /dev/null +++ b/app/models/import.rb @@ -0,0 +1,6 @@ +class Import < ApplicationRecord + belongs_to :user, dependent: :destroy + has_many :points, dependent: :destroy + + enum source: { google: 0, owntracks: 1 } +end diff --git a/app/models/point.rb b/app/models/point.rb new file mode 100644 index 00000000..3d0405e6 --- /dev/null +++ b/app/models/point.rb @@ -0,0 +1,42 @@ +class Point < ApplicationRecord + belongs_to :import, optional: true + + validates :latitude, :longitude, :tracker_id, :timestamp, :topic, presence: true + + enum battery_status: { unknown: 0, unplugged: 1, charging: 2, full: 3 }, _suffix: true + enum trigger: { + unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3, + report_location_message_event: 4, manual_event: 5, timer_based_event: 6, + settings_monitoring_event: 7 + }, _suffix: true + enum connection: { mobile: 0, wifi: 1, offline: 2 }, _suffix: true + + after_create :async_reverse_geocode + + def tracked_at + Time.at(timestamp).strftime('%Y-%m-%d %H:%M:%S') + end + + def cities_by_countries + group_by { _1.country }.compact.map { |k, v| { k => v.pluck(:city).uniq.compact } } + end + + private + + def async_reverse_geocode + ReverseGeocodingJob.perform_later(id) + end +end + + +def group_records_by_hour(records) + grouped_records = Hash.new { |hash, key| hash[key] = [] } + + records.each do |record| + # Round timestamp to the nearest hour + rounded_time = Time.at(record.timestamp).beginning_of_hour + grouped_records[rounded_time] << record + end + + grouped_records +end diff --git a/app/models/user.rb b/app/models/user.rb index 47567994..c58a7639 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -3,4 +3,7 @@ class User < ApplicationRecord # :confirmable, :lockable, :timeoutable, :trackable and :omniauthable devise :database_authenticatable, :registerable, :recoverable, :rememberable, :validatable + + has_many :imports, dependent: :destroy + has_many :points, through: :imports end diff --git a/app/services/google_maps/timeline_parser.rb b/app/services/google_maps/timeline_parser.rb new file mode 100644 index 00000000..ff568f87 --- /dev/null +++ b/app/services/google_maps/timeline_parser.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +class GoogleMaps::TimelineParser + attr_reader :file_path, :file, :json, :import_id + + def initialize(file_path, import_id = nil) + @file_path = file_path + + raise 'File not found' unless File.exist?(@file_path) + + @file = File.read(@file_path) + @json = JSON.parse(@file) + @import_id = import_id + end + + def call + points_data = parse_json + + points_data.each do |point_data| + Point.create( + latitude: point_data[:latitude], + longitude: point_data[:longitude], + timestamp: point_data[:timestamp], + raw_data: point_data[:raw_data], + topic: 'Google Maps Timeline Export', + tracker_id: 'google-maps-timeline-export', + import_id: import_id + ) + end + end + + private + + def parse_json + json['timelineObjects'].flat_map do |timeline_object| + if timeline_object['activitySegment'].present? + if timeline_object['activitySegment']['startLocation'].blank? + next if timeline_object['activitySegment']['waypointPath'].blank? + + timeline_object['activitySegment']['waypointPath']['waypoints'].map do |waypoint| + { + latitude: waypoint['latE7'].to_f / 10**7, + longitude: waypoint['lngE7'].to_f / 10**7, + timestamp: DateTime.parse(timeline_object['activitySegment']['duration']['startTimestamp']), + raw_data: timeline_object + } + end + else + { + latitude: timeline_object['activitySegment']['startLocation']['latitudeE7'].to_f / 10**7, + longitude: timeline_object['activitySegment']['startLocation']['longitudeE7'].to_f / 10**7, + timestamp: DateTime.parse(timeline_object['activitySegment']['duration']['startTimestamp']), + raw_data: timeline_object + } + end + elsif timeline_object['placeVisit'].present? + { + latitude: timeline_object['placeVisit']['location']['latitudeE7'].to_f / 10**7, + longitude: timeline_object['placeVisit']['location']['longitudeE7'].to_f / 10**7, + timestamp: DateTime.parse(timeline_object['placeVisit']['duration']['startTimestamp']), + raw_data: timeline_object + } + end + end.reject(&:blank?) + end +end diff --git a/app/services/own_tracks/export_parser.rb b/app/services/own_tracks/export_parser.rb new file mode 100644 index 00000000..b34ca603 --- /dev/null +++ b/app/services/own_tracks/export_parser.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class OwnTracks::ExportParser + attr_reader :file_path, :file, :json, :import_id + + def initialize(file_path, import_id = nil) + @file_path = file_path + + raise 'File not found' unless File.exist?(@file_path) + + @file = File.read(@file_path) + @json = JSON.parse(@file) + @import_id = import_id + end + + def call + points_data = parse_json + + points_data.each do |point_data| + Point.create( + latitude: point_data[:latitude], + longitude: point_data[:longitude], + timestamp: point_data[:timestamp], + raw_data: point_data[:raw_data], + topic: point_data[:topic], + tracker_id: point_data[:tracker_id], + import_id: import_id + ) + end + end + + private + + def parse_json + points = [] + + json.keys.each do |user| + json[user].keys.each do |devise| + json[user][devise].each { |point| points << OwnTracks::Params.new(point).call } + end + end + + points + end +end diff --git a/app/services/own_tracks/params.rb b/app/services/own_tracks/params.rb new file mode 100644 index 00000000..f22b599d --- /dev/null +++ b/app/services/own_tracks/params.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class OwnTracks::Params + attr_reader :params + + def initialize(params) + @params = params.deep_symbolize_keys + end + + def call + { + latitude: params[:lat], + longitude: params[:lon], + battery_status: battery_status, + battery: params[:batt], + ping: params[:p], + altitude: params[:alt], + accuracy: params[:acc], + vertical_accuracy: params[:vac], + velocity: params[:vel], + connection: connection, + ssid: params[:SSID], + bssid: params[:BSSID], + trigger: trigger, + tracker_id: params[:tid], + timestamp: Time.at(params[:tst].to_i), + inrids: params[:inrids], + in_regions: params[:inregions], + topic: params[:topic], + raw_data: params + } + end + + private + + def battery_status + return 'unknown' if params[:bs].nil? + + case params[:bs] + when 'u' then 'unplugged' + when 'c' then 'charging' + when 'f' then 'full' + else 'unknown' + end + end + + def trigger + return 'unknown' if params[:m].nil? + + case params[:m] + when 'p' then 'background_event' + when 'c' then 'circular_region_event' + when 'b' then 'beacon_event' + when 'r' then 'report_location_message_event' + when 'u' then 'manual_event' + when 't' then 'timer_based_event' + when 'v' then 'settings_monitoring_event' + else 'unknown' + end + end + + def connection + return 'mobile' if params[:conn].nil? + + case params[:conn] + when 'm' then 'mobile' + when 'w' then 'wifi' + when 'o' then 'offline' + else 'unknown' + end + end +end diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erb new file mode 100644 index 00000000..b3601292 --- /dev/null +++ b/app/views/imports/_form.html.erb @@ -0,0 +1,31 @@ +<%= form_with(model: import, class: "contents") do |form| %> + <% if import.errors.any? %> +
+

<%= pluralize(import.errors.count, "error") %> prohibited this import from being saved:

+ + +
+ <% end %> + + + + + +
+ <%= form.submit class: "rounded-lg py-3 px-5 bg-blue-600 text-white inline-block font-medium cursor-pointer" %> +
+<% end %> diff --git a/app/views/imports/_import.html.erb b/app/views/imports/_import.html.erb new file mode 100644 index 00000000..783ba5d3 --- /dev/null +++ b/app/views/imports/_import.html.erb @@ -0,0 +1,12 @@ +
+

+ Name: + <%= import.name %> +

+ + <% if action_name != "show" %> + <%= link_to "Show this import", import, class: "rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> + <%= link_to "Edit this import", edit_import_path(import), class: "rounded-lg py-3 ml-2 px-5 bg-gray-100 inline-block font-medium" %> +
+ <% end %> +
diff --git a/app/views/imports/edit.html.erb b/app/views/imports/edit.html.erb new file mode 100644 index 00000000..7a2cdd7d --- /dev/null +++ b/app/views/imports/edit.html.erb @@ -0,0 +1,8 @@ +
+

Editing import

+ + <%= render "form", import: @import %> + + <%= link_to "Show this import", @import, class: "ml-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> + <%= link_to "Back to imports", imports_path, class: "ml-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
diff --git a/app/views/imports/index.html.erb b/app/views/imports/index.html.erb new file mode 100644 index 00000000..bbf0c6c5 --- /dev/null +++ b/app/views/imports/index.html.erb @@ -0,0 +1,10 @@ +
+
+

Imports

+ <%= link_to "New import", new_import_path, class: "rounded-lg py-3 px-5 bg-blue-600 text-white block font-medium" %> +
+ +
+ <%= render @imports %> +
+
diff --git a/app/views/imports/new.html.erb b/app/views/imports/new.html.erb new file mode 100644 index 00000000..e1de6011 --- /dev/null +++ b/app/views/imports/new.html.erb @@ -0,0 +1,7 @@ +
+

New import

+ + <%= render "form", import: @import %> + + <%= link_to "Back to imports", imports_path, class: "ml-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
diff --git a/app/views/imports/show.html.erb b/app/views/imports/show.html.erb new file mode 100644 index 00000000..83dc2e69 --- /dev/null +++ b/app/views/imports/show.html.erb @@ -0,0 +1,15 @@ +
+
+ <% if notice.present? %> +

<%= notice %>

+ <% end %> + + <%= render @import %> + + <%= link_to "Edit this import", edit_import_path(@import), class: "mt-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
+ <%= button_to "Destroy this import", import_path(@import), method: :delete, class: "mt-2 rounded-lg py-3 px-5 bg-gray-100 font-medium" %> +
+ <%= link_to "Back to imports", imports_path, class: "ml-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
+
diff --git a/app/views/points/_table.html.erb b/app/views/points/_table.html.erb new file mode 100644 index 00000000..66af236a --- /dev/null +++ b/app/views/points/_table.html.erb @@ -0,0 +1,61 @@ +

Points Table

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <% @points.each do |point| %> + + + + + + + + + + + + + + + + + + + + + + + + + + <% end %> + +
IDBattery StatusPingBatteryTracked IDTopicAltitudeVelocityTriggerBSSIDSSIDConnectionVertical AccuracyAccuracyTimestampModeLatitudeLongitudeInridsIn RegionsRaw DataTracker IDCreated AtUpdated At
<%= point.id %><%= point.battery_status %><%= point.ping %><%= point.battery %><%= point.tracker_id %><%= point.topic %><%= point.altitude %><%= point.velocity %><%= point.trigger %><%= point.bssid %><%= point.ssid %><%= point.connection %><%= point.vertical_accuracy %><%= point.accuracy %><%= point.timestamp %><%= point.mode %><%= point.latitude %><%= point.longitude %><%= point.inrids %><%= point.in_regions %><%= point.raw_data %><%= point.created_at %><%= point.updated_at %>
diff --git a/app/views/points/index.html.erb b/app/views/points/index.html.erb new file mode 100644 index 00000000..160d4cbd --- /dev/null +++ b/app/views/points/index.html.erb @@ -0,0 +1,3 @@ +<%= js_map(@coordinates, style: "mapbox://styles/mapbox/outdoors-v12", trail: true) %> + +<%#= render 'points/table', points: @points %> diff --git a/app/views/shared/_navbar.html.erb b/app/views/shared/_navbar.html.erb index 8623931d..f73d3f7b 100644 --- a/app/views/shared/_navbar.html.erb +++ b/app/views/shared/_navbar.html.erb @@ -5,7 +5,8 @@ <%= link_to 'Dawarich', root_path, class: 'btn btn-ghost normal-case text-xl'%> @@ -17,23 +18,26 @@
<%# menu menu-sm dropdown-content mt-3 z-[1] p-2 shadow bg-base-100 rounded-box w-52 %> -