kevinsivic/dawarich
1
0
Fork 0
mirror of https://github.com/Freika/dawarich.git synced 2026-01-10 17:21:38 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1951 from Freika/feature/kml

Browse source 
Feature/kml
This commit is contained in:
Evgenii Burmakin 2025-11-14 18:21:01 +01:00 committed by GitHub
commit cfe5a77a47
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 859 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
CHANGELOG.md
View file

@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
## Added
- Support for KML file uploads. #350
- Added a commented line in the `docker-compose.yml` file to use an alternative PostGIS image for ARM architecture.
## Fixed
- The map settings panel is now scrollable
## Changed
- Internal redis settings updated to implement support for connecting to Redis via unix socket. #1706
# [0.35.1] - 2025-11-09
## Fixed

4
app/assets/builds/tailwind.css
View file

File diff suppressed because one or more lines are too long

8
app/assets/stylesheets/application.css
View file

@ -27,9 +27,13 @@
/* Style for the settings panel */
.leaflet-settings-panel {
background-color: white;
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
box-shadow: 0 1px 4px rgba(0, 0, 0, 0.3);
position: absolute !important;
top: 10px !important;
left: 60px !important;
transform: none;
z-index: 1000;
}
.leaflet-settings-panel label {

177
app/javascript/controllers/maps_controller.js
View file

@ -955,100 +955,141 @@ export default class extends BaseController {
// Form HTML
div.innerHTML = `
<form id="settings-form" style="overflow-y: auto; max-height: 70vh; width: 12rem; padding-right: 5px;">
<label for="route-opacity">Route Opacity, %</label>
<div class="join">
<input type="number" class="input input-ghost join-item focus:input-ghost input-xs input-bordered w-full max-w-xs" id="route-opacity" name="route_opacity" min="10" max="100" step="10" value="${Math.round(this.routeOpacity * 100)}">
<label for="route_opacity_info" class="btn-xs join-item ">?</label>
<form id="settings-form" class="space-y-3">
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Route Opacity, %</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="route-opacity" name="route_opacity" min="10" max="100" step="10" value="${Math.round(this.routeOpacity * 100)}">
<label for="route_opacity_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="fog_of_war_meters">Fog of War radius</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="fog_of_war_meters" name="fog_of_war_meters" min="5" max="200" step="1" value="${this.clearFogRadius}">
<label for="fog_of_war_meters_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Fog of War radius</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="fog_of_war_meters" name="fog_of_war_meters" min="5" max="200" step="1" value="${this.clearFogRadius}">
<label for="fog_of_war_meters_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="fog_of_war_threshold">Seconds between Fog of War lines</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="fog_of_war_threshold" name="fog_of_war_threshold" step="1" value="${this.userSettings.fog_of_war_threshold}">
<label for="fog_of_war_threshold_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Fog of War threshold</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="fog_of_war_threshold" name="fog_of_war_threshold" step="1" value="${this.userSettings.fog_of_war_threshold}">
<label for="fog_of_war_threshold_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="meters_between_routes">Meters between routes</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="meters_between_routes" name="meters_between_routes" step="1" value="${this.userSettings.meters_between_routes}">
<label for="meters_between_routes_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Meters between routes</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="meters_between_routes" name="meters_between_routes" step="1" value="${this.userSettings.meters_between_routes}">
<label for="meters_between_routes_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="minutes_between_routes">Minutes between routes</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="minutes_between_routes" name="minutes_between_routes" step="1" value="${this.userSettings.minutes_between_routes}">
<label for="minutes_between_routes_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Minutes between routes</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="minutes_between_routes" name="minutes_between_routes" step="1" value="${this.userSettings.minutes_between_routes}">
<label for="minutes_between_routes_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="time_threshold_minutes">Time threshold minutes</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="time_threshold_minutes" name="time_threshold_minutes" step="1" value="${this.userSettings.time_threshold_minutes}">
<label for="time_threshold_minutes_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Time threshold minutes</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="time_threshold_minutes" name="time_threshold_minutes" step="1" value="${this.userSettings.time_threshold_minutes}">
<label for="time_threshold_minutes_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="merge_threshold_minutes">Merge threshold minutes</label>
<div class="join">
<input type="number" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="merge_threshold_minutes" name="merge_threshold_minutes" step="1" value="${this.userSettings.merge_threshold_minutes}">
<label for="merge_threshold_minutes_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Merge threshold minutes</span>
</label>
<div class="join join-horizontal w-full">
<input type="number" class="input input-bordered input-sm join-item flex-1" id="merge_threshold_minutes" name="merge_threshold_minutes" step="1" value="${this.userSettings.merge_threshold_minutes}">
<label for="merge_threshold_minutes_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
</div>
<label for="points_rendering_mode">
Points rendering mode
<label for="points_rendering_mode_info" class="btn-xs join-item inline">?</label>
</label>
<label for="raw">
<input type="radio" id="raw" name="points_rendering_mode" class='w-4' style="width: 20px;" value="raw" ${this.pointsRenderingModeChecked('raw')} />
Raw
</label>
<label for="simplified">
<input type="radio" id="simplified" name="points_rendering_mode" class='w-4' style="width: 20px;" value="simplified" ${this.pointsRenderingModeChecked('simplified')}/>
Simplified
</label>
<label for="live_map_enabled">
Live Map
<label for="live_map_enabled_info" class="btn-xs join-item inline">?</label>
<input type="checkbox" id="live_map_enabled" name="live_map_enabled" class='w-4' style="width: 20px;" value="false" ${this.liveMapEnabledChecked(true)} />
</label>
<label for="speed_colored_routes">
Speed-colored routes
<label for="speed_colored_routes_info" class="btn-xs join-item inline">?</label>
<input type="checkbox" id="speed_colored_routes" name="speed_colored_routes" class='w-4' style="width: 20px;" ${this.speedColoredRoutesChecked()} />
</label>
<label for="speed_color_scale">Speed color scale</label>
<div class="join">
<input type="text" class="join-item input input-ghost focus:input-ghost input-xs input-bordered w-full max-w-xs" id="speed_color_scale" name="speed_color_scale" min="5" max="100" step="1" value="${this.speedColorScale}">
<label for="speed_color_scale_info" class="btn-xs join-item">?</label>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Points rendering mode</span>
<label for="points_rendering_mode_info" class="btn btn-xs btn-ghost cursor-pointer">?</label>
</label>
<div class="flex flex-col gap-2">
<label class="label cursor-pointer justify-start gap-2 py-1">
<input type="radio" id="raw" name="points_rendering_mode" class="radio radio-sm" value="raw" ${this.pointsRenderingModeChecked('raw')} />
<span class="label-text text-xs">Raw</span>
</label>
<label class="label cursor-pointer justify-start gap-2 py-1">
<input type="radio" id="simplified" name="points_rendering_mode" class="radio radio-sm" value="simplified" ${this.pointsRenderingModeChecked('simplified')} />
<span class="label-text text-xs">Simplified</span>
</label>
</div>
</div>
<button type="button" id="edit-gradient-btn" class="btn btn-xs mt-2">Edit Scale</button>
<hr>
<div class="form-control">
<label class="label cursor-pointer py-1">
<span class="label-text text-xs">Live Map</span>
<div class="flex items-center gap-1">
<label for="live_map_enabled_info" class="btn btn-xs btn-ghost cursor-pointer">?</label>
<input type="checkbox" id="live_map_enabled" name="live_map_enabled" class="checkbox checkbox-sm" ${this.liveMapEnabledChecked(true)} />
</div>
</label>
</div>
<button type="submit" class="btn btn-xs mt-2">Update</button>
<div class="form-control">
<label class="label cursor-pointer py-1">
<span class="label-text text-xs">Speed-colored routes</span>
<div class="flex items-center gap-1">
<label for="speed_colored_routes_info" class="btn btn-xs btn-ghost cursor-pointer">?</label>
<input type="checkbox" id="speed_colored_routes" name="speed_colored_routes" class="checkbox checkbox-sm" ${this.speedColoredRoutesChecked()} />
</div>
</label>
</div>
<div class="form-control">
<label class="label py-1">
<span class="label-text text-xs">Speed color scale</span>
</label>
<div class="join join-horizontal w-full">
<input type="text" class="input input-bordered input-sm join-item flex-1" id="speed_color_scale" name="speed_color_scale" value="${this.speedColorScale}">
<label for="speed_color_scale_info" class="btn btn-sm btn-ghost join-item cursor-pointer">?</label>
</div>
<button type="button" id="edit-gradient-btn" class="btn btn-sm mt-2 w-full">Edit Colors</button>
</div>
<div class="divider my-2"></div>
<button type="submit" class="btn btn-sm btn-primary w-full">Update</button>
</form>
`;
// Style the panel with theme-aware styling
applyThemeToPanel(div, this.userTheme);
div.style.padding = '10px';
div.style.width = '220px';
div.style.maxHeight = 'calc(60vh - 20px)';
div.style.overflowY = 'auto';
// Prevent map interactions when interacting with the form
L.DomEvent.disableClickPropagation(div);
L.DomEvent.disableScrollPropagation(div);
// Attach event listener to the "Edit Gradient" button:
const editBtn = div.querySelector("#edit-gradient-btn");

2
app/models/import.rb
View file

@ -22,7 +22,7 @@ class Import < ApplicationRecord
enum :source, {
google_semantic_history: 0, owntracks: 1, google_records: 2,
google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6, photoprism_api: 7,
user_data_archive: 8
user_data_archive: 8, kml: 9
}, allow_nil: true
def process!

1
app/services/imports/create.rb
View file

@ -58,6 +58,7 @@ class Imports::Create
when 'google_records' then GoogleMaps::RecordsStorageImporter
when 'owntracks' then OwnTracks::Importer
when 'gpx' then Gpx::TrackImporter
when 'kml' then Kml::Importer
when 'geojson' then Geojson::Importer
when 'immich_api', 'photoprism_api' then Photos::Importer
else

17
app/services/imports/source_detector.rb
View file

@ -71,6 +71,7 @@ class Imports::SourceDetector
def detect_source
return :gpx if gpx_file?
return :kml if kml_file?
return :owntracks if owntracks_file?
json_data = parse_json
@ -116,6 +117,22 @@ class Imports::SourceDetector
) && content_to_check.include?('<gpx')
end
def kml_file?
return false unless filename&.downcase&.end_with?('.kml', '.kmz')
content_to_check =
if file_path && File.exist?(file_path)
# Read first 1KB for KML detection
File.open(file_path, 'rb') { |f| f.read(1024) }
else
file_content
end
(
content_to_check.strip.start_with?('<?xml') ||
content_to_check.strip.start_with?('<kml')
) && content_to_check.include?('<kml')
end
def owntracks_file?
return false unless filename

234
app/services/kml/importer.rb Normal file
View file

@ -0,0 +1,234 @@
# frozen_string_literal: true
require 'rexml/document'
class Kml::Importer
include Imports::Broadcaster
include Imports::FileLoader
attr_reader :import, :user_id, :file_path
def initialize(import, user_id, file_path = nil)
@import = import
@user_id = user_id
@file_path = file_path
end
def call
file_content = load_file_content
doc = REXML::Document.new(file_content)
points_data = []
# Process all Placemarks which can contain various geometry types
REXML::XPath.each(doc, '//Placemark') do |placemark|
points_data.concat(parse_placemark(placemark))
end
# Process gx:Track elements (Google Earth extensions for GPS tracks)
REXML::XPath.each(doc, '//gx:Track') do |track|
points_data.concat(parse_gx_track(track))
end
points_data.compact!
return if points_data.empty?
# Process in batches to avoid memory issues with large files
points_data.each_slice(1000) do |batch|
bulk_insert_points(batch)
end
end
private
def parse_placemark(placemark)
points = []
timestamp = extract_timestamp(placemark)
# Handle Point geometry
point_node = REXML::XPath.first(placemark, './/Point/coordinates')
if point_node
coords = parse_coordinates(point_node.text)
points << build_point(coords.first, timestamp, placemark) if coords.any?
end
# Handle LineString geometry (tracks/routes)
linestring_node = REXML::XPath.first(placemark, './/LineString/coordinates')
if linestring_node
coords = parse_coordinates(linestring_node.text)
coords.each do |coord|
points << build_point(coord, timestamp, placemark)
end
end
# Handle MultiGeometry (can contain multiple Points, LineStrings, etc.)
REXML::XPath.each(placemark, './/MultiGeometry//coordinates') do |coords_node|
coords = parse_coordinates(coords_node.text)
coords.each do |coord|
points << build_point(coord, timestamp, placemark)
end
end
points.compact
end
def parse_gx_track(track)
# Google Earth Track extension with coordinated when/coord pairs
points = []
timestamps = []
REXML::XPath.each(track, './/when') do |when_node|
timestamps << when_node.text.strip
end
coordinates = []
REXML::XPath.each(track, './/gx:coord') do |coord_node|
coordinates << coord_node.text.strip
end
# Match timestamps with coordinates
[timestamps.size, coordinates.size].min.times do |i|
begin
time = Time.parse(timestamps[i]).to_i
coord_parts = coordinates[i].split(/\s+/)
next if coord_parts.size < 2
lng, lat, alt = coord_parts.map(&:to_f)
points << {
lonlat: "POINT(#{lng} #{lat})",
altitude: alt&.to_i || 0,
timestamp: time,
import_id: import.id,
velocity: 0.0,
raw_data: { source: 'gx_track', index: i },
user_id: user_id,
created_at: Time.current,
updated_at: Time.current
}
rescue StandardError => e
Rails.logger.warn("Failed to parse gx:Track point at index #{i}: #{e.message}")
next
end
end
points
end
def parse_coordinates(coord_text)
# KML coordinates format: "longitude,latitude[,altitude] ..."
# Multiple coordinates separated by whitespace
return [] if coord_text.blank?
coord_text.strip.split(/\s+/).map do |coord_str|
parts = coord_str.split(',')
next if parts.size < 2
{
lng: parts[0].to_f,
lat: parts[1].to_f,
alt: parts[2]&.to_f || 0.0
}
end.compact
end
def extract_timestamp(placemark)
# Try TimeStamp first
timestamp_node = REXML::XPath.first(placemark, './/TimeStamp/when')
return Time.parse(timestamp_node.text).to_i if timestamp_node
# Try TimeSpan begin
timespan_begin = REXML::XPath.first(placemark, './/TimeSpan/begin')
return Time.parse(timespan_begin.text).to_i if timespan_begin
# Try TimeSpan end as fallback
timespan_end = REXML::XPath.first(placemark, './/TimeSpan/end')
return Time.parse(timespan_end.text).to_i if timespan_end
# Default to import creation time if no timestamp found
import.created_at.to_i
rescue StandardError => e
Rails.logger.warn("Failed to parse timestamp: #{e.message}")
import.created_at.to_i
end
def build_point(coord, timestamp, placemark)
return if coord[:lat].blank? || coord[:lng].blank?
{
lonlat: "POINT(#{coord[:lng]} #{coord[:lat]})",
altitude: coord[:alt].to_i,
timestamp: timestamp,
import_id: import.id,
velocity: extract_velocity(placemark),
raw_data: extract_extended_data(placemark),
user_id: user_id,
created_at: Time.current,
updated_at: Time.current
}
end
def extract_velocity(placemark)
# Try to extract speed from ExtendedData
speed_node = REXML::XPath.first(placemark, ".//Data[@name='speed']/value") ||
REXML::XPath.first(placemark, ".//Data[@name='Speed']/value") ||
REXML::XPath.first(placemark, ".//Data[@name='velocity']/value")
return speed_node.text.to_f.round(1) if speed_node
0.0
rescue StandardError
0.0
end
def extract_extended_data(placemark)
data = {}
# Extract name if present
name_node = REXML::XPath.first(placemark, './/name')
data['name'] = name_node.text.strip if name_node
# Extract description if present
desc_node = REXML::XPath.first(placemark, './/description')
data['description'] = desc_node.text.strip if desc_node
# Extract all ExtendedData/Data elements
REXML::XPath.each(placemark, './/ExtendedData/Data') do |data_node|
name = data_node.attributes['name']
value_node = REXML::XPath.first(data_node, './value')
data[name] = value_node.text if name && value_node
end
data
rescue StandardError => e
Rails.logger.warn("Failed to extract extended data: #{e.message}")
{}
end
def bulk_insert_points(batch)
unique_batch = batch.uniq { |record| [record[:lonlat], record[:timestamp], record[:user_id]] }
# rubocop:disable Rails/SkipsModelValidations
Point.upsert_all(
unique_batch,
unique_by: %i[lonlat timestamp user_id],
returning: false,
on_duplicate: :skip
)
# rubocop:enable Rails/SkipsModelValidations
broadcast_import_progress(import, unique_batch.size)
rescue StandardError => e
create_notification("Failed to process KML file: #{e.message}")
end
def create_notification(message)
Notification.create!(
user_id: user_id,
title: 'KML Import Error',
content: message,
kind: :error
)
end
end

1
app/views/imports/_form.html.erb
View file

@ -7,6 +7,7 @@
<li><strong>✅ GPX:</strong> Track files (.gpx)</li>
<li><strong>✅ GeoJSON:</strong> Feature collections (.json)</li>
<li><strong>✅ OwnTracks:</strong> Recorder files (.rec)</li>
<li><strong>✅ KML:</strong> KML files (.kml)</li>
</ul>
<div class="text-xs text-gray-500 mt-2">
File format is automatically detected during upload.

2
app/views/map/_settings_modals.html.erb
View file

@ -184,7 +184,7 @@
Here you can set a custom color scale for speed colored routes. It uses color stops at specified km/h values and creates a gradient from it. The default value is <code>0:#00ff00|15:#00ffff|30:#ff00ff|50:#ffff00|100:#ff3300</code>
</p>
<p class="py-4">
You can also use the 'Edit Scale' button to edit it using an UI.
You can also use the 'Edit Colors' button to edit it using an UI.
</p>
</div>
<label class="modal-backdrop" for="speed_color_scale_info">Close</label>

3
config/cable.yml
View file

@ -1,6 +1,7 @@
default: &default
adapter: redis
url: <%= "#{ENV.fetch("REDIS_URL", "redis://localhost:6379")}/#{ENV.fetch('RAILS_WS_DB', 2)}" %>
url: <%= "#{ENV.fetch("REDIS_URL", "redis://localhost:6379")}" %>
db: <%= ENV.fetch('RAILS_WS_DB', 2) %>
development:
<<: *default

5
config/environments/development.rb
View file

@ -26,7 +26,10 @@ Rails.application.configure do
# Enable/disable caching. By default caching is disabled.
# Run rails dev:cache to toggle caching.
config.cache_store = :redis_cache_store, { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}" }
config.cache_store = :redis_cache_store, {
url: ENV['REDIS_URL'],
db: ENV.fetch('RAILS_CACHE_DB', 0)
}
if Rails.root.join('tmp/caching-dev.txt').exist?
config.action_controller.perform_caching = true

5
config/environments/production.rb
View file

@ -73,7 +73,10 @@ Rails.application.configure do
config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info')
# Use a different cache store in production.
config.cache_store = :redis_cache_store, { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}" }
config.cache_store = :redis_cache_store, {
url: ENV['REDIS_URL'],
db: ENV.fetch('RAILS_CACHE_DB', 0)
}
# Use a real queuing backend for Active Job (and separate queues per environment).
config.active_job.queue_adapter = :sidekiq

2
config/initializers/geocoder.rb
View file

@ -4,7 +4,7 @@ settings = {
debug_mode: true,
timeout: 5,
units: :km,
cache: Redis.new(url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_CACHE_DB', 0)}"),
cache: Redis.new(url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_CACHE_DB', 0)),
always_raise: :all,
http_headers: {
'User-Agent' => "Dawarich #{APP_VERSION} (https://dawarich.app)"

2
config/initializers/sidekiq.rb
View file

@ -1,7 +1,7 @@
# frozen_string_literal: true
Sidekiq.configure_server do |config|
config.redis = { url: "#{ENV['REDIS_URL']}/#{ENV.fetch('RAILS_JOB_QUEUE_DB', 1)}" }
config.redis = { url: ENV['REDIS_URL'], db: ENV.fetch('RAILS_JOB_QUEUE_DB', 1) }
config.logger = Sidekiq::Logger.new($stdout)
if ENV['PROMETHEUS_EXPORTER_ENABLED'].to_s == 'true'

2
docker/docker-compose.yml
View file

@ -20,12 +20,12 @@ services:
dawarich_db:
image: postgis/postgis:17-3.5-alpine
# image: imresamu/postgis:17-3.5-alpine # If you're on ARM architecture, use this image instead
shm_size: 1G
container_name: dawarich_db
volumes:
- dawarich_db_data:/var/lib/postgresql/data
- dawarich_shared:/var/shared
# - ./postgresql.conf:/etc/postgresql/postgresql.conf # Optional, uncomment if you want to use a custom config
networks:
- dawarich
environment:

36
postgresql.conf.example
View file

@ -1,36 +0,0 @@
listen_addresses = '*'
max_connections = 50
shared_buffers = 512MB
work_mem = 128MB
maintenance_work_mem = 128MB
dynamic_shared_memory_type = posix
checkpoint_timeout = 10min # range 30s-1d
max_wal_size = 2GB
min_wal_size = 80MB
max_parallel_workers_per_gather = 4
log_min_duration_statement = 500 # -1 is disabled, 0 logs all statements
# -1 disables, 0 logs all temp files
log_timezone = 'UTC'
autovacuum_vacuum_scale_factor = 0.05 # fraction of table size before vacuum
autovacuum_analyze_scale_factor = 0.05 # fraction of table size before analyze
datestyle = 'iso, dmy'
timezone = 'UTC'
lc_messages = 'en_US.utf8' # locale for system error message
# strings
lc_monetary = 'en_US.utf8' # locale for monetary formatting
lc_numeric = 'en_US.utf8' # locale for number formatting
lc_time = 'en_US.utf8' # locale for time formatting
default_text_search_config = 'pg_catalog.english'

27
spec/fixtures/files/kml/extended_data.kml vendored Normal file
View file

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>Extended Data Example</name>
<Placemark>
<name>Location with Speed</name>
<description>A location with extended data including speed</description>
<TimeStamp>
<when>2024-01-19T11:30:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0841,37.4220,10</coordinates>
</Point>
<ExtendedData>
<Data name="speed">
<value>5.5</value>
</Data>
<Data name="accuracy">
<value>10</value>
</Data>
<Data name="battery">
<value>85</value>
</Data>
</ExtendedData>
</Placemark>
</Document>
</kml>

19
spec/fixtures/files/kml/gx_track.kml vendored Normal file
View file

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2" xmlns:gx="http://www.google.com/kml/ext/2.2">
<Document>
<name>Google Earth Track</name>
<Placemark>
<name>GPS Track</name>
<gx:Track>
<when>2024-01-20T08:00:00Z</when>
<when>2024-01-20T08:01:00Z</when>
<when>2024-01-20T08:02:00Z</when>
<when>2024-01-20T08:03:00Z</when>
<gx:coord>-122.0841 37.4220 10</gx:coord>
<gx:coord>-122.0851 37.4230 12</gx:coord>
<gx:coord>-122.0861 37.4240 14</gx:coord>
<gx:coord>-122.0871 37.4250 16</gx:coord>
</gx:Track>
</Placemark>
</Document>
</kml>

24
spec/fixtures/files/kml/invalid_coordinates.kml vendored Normal file
View file

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>Invalid Coordinates</name>
<Placemark>
<name>No Coordinates</name>
<TimeStamp>
<when>2024-01-23T10:00:00Z</when>
</TimeStamp>
<Point>
<coordinates></coordinates>
</Point>
</Placemark>
<Placemark>
<name>Only Longitude</name>
<TimeStamp>
<when>2024-01-23T11:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0841</coordinates>
</Point>
</Placemark>
</Document>
</kml>

46
spec/fixtures/files/kml/large_track.kml vendored Normal file
View file

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>Large Track for Batch Testing</name>
<Placemark>
<name>Long Track</name>
<TimeStamp>
<when>2024-01-25T00:00:00Z</when>
</TimeStamp>
<LineString>
<coordinates>
-122.0841,37.4220,10
-122.0842,37.4221,10
-122.0843,37.4222,10
-122.0844,37.4223,10
-122.0845,37.4224,10
-122.0846,37.4225,10
-122.0847,37.4226,10
-122.0848,37.4227,10
-122.0849,37.4228,10
-122.0850,37.4229,10
</coordinates>
</LineString>
</Placemark>
<Placemark>
<name>Another Long Track</name>
<TimeStamp>
<when>2024-01-25T12:00:00Z</when>
</TimeStamp>
<LineString>
<coordinates>
-122.0851,37.4230,12
-122.0852,37.4231,12
-122.0853,37.4232,12
-122.0854,37.4233,12
-122.0855,37.4234,12
-122.0856,37.4235,12
-122.0857,37.4236,12
-122.0858,37.4237,12
-122.0859,37.4238,12
-122.0860,37.4239,12
</coordinates>
</LineString>
</Placemark>
</Document>
</kml>

21
spec/fixtures/files/kml/linestring_track.kml vendored Normal file
View file

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>LineString Track</name>
<Placemark>
<name>My Track</name>
<TimeStamp>
<when>2024-01-16T10:00:00Z</when>
</TimeStamp>
<LineString>
<coordinates>
-122.0841,37.4220,10
-122.0851,37.4230,12
-122.0861,37.4240,14
-122.0871,37.4250,16
-122.0881,37.4260,18
</coordinates>
</LineString>
</Placemark>
</Document>
</kml>

28
spec/fixtures/files/kml/multigeometry.kml vendored Normal file
View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>MultiGeometry Example</name>
<Placemark>
<name>Multiple Geometries</name>
<TimeStamp>
<when>2024-01-18T15:00:00Z</when>
</TimeStamp>
<MultiGeometry>
<Point>
<coordinates>-122.0841,37.4220,10</coordinates>
</Point>
<Point>
<coordinates>-122.0851,37.4230,12</coordinates>
</Point>
<LineString>
<coordinates>
-122.0861,37.4240,14
-122.0871,37.4250,16
-122.0881,37.4260,18
-122.0891,37.4270,20
</coordinates>
</LineString>
</MultiGeometry>
</Placemark>
</Document>
</kml>

51
spec/fixtures/files/kml/nested_folders.kml vendored Normal file
View file

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>Nested Folders</name>
<Folder>
<name>Trip 1</name>
<Placemark>
<name>Start Point</name>
<TimeStamp>
<when>2024-01-21T08:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0841,37.4220,10</coordinates>
</Point>
</Placemark>
<Folder>
<name>Day 1</name>
<Placemark>
<name>Checkpoint 1</name>
<TimeStamp>
<when>2024-01-21T12:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0851,37.4230,12</coordinates>
</Point>
</Placemark>
</Folder>
</Folder>
<Folder>
<name>Trip 2</name>
<Placemark>
<name>Location A</name>
<TimeStamp>
<when>2024-01-22T10:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0861,37.4240,14</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Location B</name>
<TimeStamp>
<when>2024-01-22T14:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0871,37.4250,16</coordinates>
</Point>
</Placemark>
</Folder>
</Document>
</kml>

33
spec/fixtures/files/kml/points_with_timestamps.kml vendored Normal file
View file

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>Points with Timestamps</name>
<Placemark>
<name>Location 1</name>
<TimeStamp>
<when>2024-01-15T12:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0841,37.4220,10</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Location 2</name>
<TimeStamp>
<when>2024-01-15T13:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0851,37.4230,15</coordinates>
</Point>
</Placemark>
<Placemark>
<name>Location 3</name>
<TimeStamp>
<when>2024-01-15T14:00:00Z</when>
</TimeStamp>
<Point>
<coordinates>-122.0861,37.4240,20</coordinates>
</Point>
</Placemark>
</Document>
</kml>

16
spec/fixtures/files/kml/timespan.kml vendored Normal file
View file

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<name>TimeSpan Example</name>
<Placemark>
<name>Visit Duration</name>
<TimeSpan>
<begin>2024-01-10T09:00:00Z</begin>
<end>2024-01-10T17:00:00Z</end>
</TimeSpan>
<Point>
<coordinates>-122.0841,37.4220,10</coordinates>
</Point>
</Placemark>
</Document>
</kml>

3
spec/models/import_spec.rb
View file

@ -115,7 +115,8 @@ RSpec.describe Import, type: :model do
immich_api: 5,
geojson: 6,
photoprism_api: 7,
user_data_archive: 8
user_data_archive: 8,
kml: 9
)
end
end

24
spec/services/imports/source_detector_spec.rb
View file

@ -74,6 +74,15 @@ RSpec.describe Imports::SourceDetector do
end
end
context 'with KML file' do
let(:file_content) { file_fixture('kml/points_with_timestamps.kml').read }
let(:filename) { 'test.kml' }
it 'detects kml format' do
expect(detector.detect_source).to eq(:kml)
end
end
context 'with invalid JSON' do
let(:file_content) { 'invalid json content' }
@ -145,6 +154,15 @@ RSpec.describe Imports::SourceDetector do
expect(detector.detect_source).to eq(:geojson)
end
end
context 'with KML file' do
let(:fixture_path) { file_fixture('kml/points_with_timestamps.kml').to_s }
it 'detects source correctly from file path' do
detector = described_class.new_from_file_header(fixture_path)
expect(detector.detect_source).to eq(:kml)
end
end
end
describe 'detection accuracy with real fixture files' do
@ -170,5 +188,11 @@ RSpec.describe Imports::SourceDetector do
include_examples 'detects format correctly', :gpx, 'gpx/arc_example.gpx'
include_examples 'detects format correctly', :gpx, 'gpx/garmin_example.gpx'
include_examples 'detects format correctly', :gpx, 'gpx/gpx_track_multiple_segments.gpx'
# Test KML files
include_examples 'detects format correctly', :kml, 'kml/points_with_timestamps.kml'
include_examples 'detects format correctly', :kml, 'kml/linestring_track.kml'
include_examples 'detects format correctly', :kml, 'kml/gx_track.kml'
include_examples 'detects format correctly', :kml, 'kml/multigeometry.kml'
end
end

168
spec/services/kml/importer_spec.rb Normal file
View file

@ -0,0 +1,168 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Kml::Importer do
describe '#call' do
subject(:parser) { described_class.new(import, user.id, file_path).call }
let(:user) { create(:user) }
let(:import) { create(:import, user:, name: 'test.kml', source: 'kml') }
context 'when file has Point placemarks with timestamps' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/points_with_timestamps.kml').to_s }
it 'creates points' do
expect { parser }.to change(Point, :count).by(3)
end
it 'creates points with correct data' do
parser
point = user.points.order(:timestamp).first
expect(point.lat).to eq(37.4220)
expect(point.lon).to eq(-122.0841)
expect(point.altitude).to eq(10)
expect(point.timestamp).to eq(Time.zone.parse('2024-01-15T12:00:00Z').to_i)
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).at_least(1).time
parser
end
end
context 'when file has LineString (track)' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/linestring_track.kml').to_s }
it 'creates points from linestring coordinates' do
expect { parser }.to change(Point, :count).by(5)
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).at_least(1).time
parser
end
end
context 'when file has gx:Track (Google Earth extension)' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/gx_track.kml').to_s }
it 'creates points from gx:Track with coordinated when/coord pairs' do
expect { parser }.to change(Point, :count).by(4)
end
it 'creates points with correct timestamps' do
parser
points = user.points.order(:timestamp)
expect(points.first.timestamp).to eq(Time.zone.parse('2024-01-20T08:00:00Z').to_i)
expect(points.last.timestamp).to eq(Time.zone.parse('2024-01-20T08:03:00Z').to_i)
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).at_least(1).time
parser
end
end
context 'when file has MultiGeometry' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/multigeometry.kml').to_s }
it 'creates points from all geometries in MultiGeometry' do
expect { parser }.to change(Point, :count).by(6)
end
it 'broadcasts importing progress' do
expect_any_instance_of(Imports::Broadcaster).to receive(:broadcast_import_progress).at_least(1).time
parser
end
end
context 'when file has ExtendedData with speed' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/extended_data.kml').to_s }
it 'creates points with velocity from ExtendedData' do
parser
point = user.points.first
expect(point.velocity).to eq('5.5')
end
it 'stores extended data in raw_data' do
parser
point = user.points.first
expect(point.raw_data['name']).to be_present
expect(point.raw_data['description']).to be_present
end
end
context 'when file has TimeSpan' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/timespan.kml').to_s }
it 'uses TimeSpan begin as timestamp' do
parser
point = user.points.first
expect(point.timestamp).to eq(Time.zone.parse('2024-01-10T09:00:00Z').to_i)
end
end
context 'when file has nested folders' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/nested_folders.kml').to_s }
it 'processes all placemarks regardless of nesting' do
expect { parser }.to change(Point, :count).by(4)
end
end
context 'when coordinates are missing required fields' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/invalid_coordinates.kml').to_s }
it 'skips invalid coordinates' do
expect { parser }.not_to change(Point, :count)
end
end
context 'when processing large file in batches' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/large_track.kml').to_s }
it 'processes points' do
expect { parser }.to change(Point, :count).by(20)
end
end
context 'when import fails' do
let(:file_path) { Rails.root.join('spec/fixtures/files/kml/points_with_timestamps.kml').to_s }
before do
allow(Point).to receive(:upsert_all).and_raise(StandardError.new('Database error'))
end
it 'creates an error notification' do
expect { parser }.to change(Notification, :count).by(1)
end
it 'creates notification with error details' do
parser
notification = Notification.last
expect(notification.user_id).to eq(user.id)
expect(notification.title).to eq('KML Import Error')
expect(notification.kind).to eq('error')
expect(notification.content).to include('Database error')
end
end
end
end