From ba2059ae15f8e7f8b8d8576e200c485a8fbf096c Mon Sep 17 00:00:00 2001 From: Roman Chesnokov Date: Thu, 14 May 2026 15:26:59 +0500 Subject: [PATCH] feat(phase-0,phase-2): bootstrap DB schemas + Metabase/NocoDB compose + work_types seed --- .env.example | 30 ++++++++ docker-compose.yml | 34 +++++++++ infra/init-bit-flight-deck-db.sql | 15 ++++ sql/migrations/001_schemas.sql | 15 ++++ sql/migrations/002_core_employee.sql | 48 +++++++++++++ sql/migrations/003_core_project.sql | 37 ++++++++++ sql/migrations/004_core_work_log.sql | 29 ++++++++ sql/migrations/005_core_task.sql | 23 ++++++ sql/migrations/006_core_deal.sql | 33 +++++++++ sql/migrations/007_core_identity_map.sql | 19 +++++ sql/migrations/008_raw_schemas.sql | 89 ++++++++++++++++++++++++ sql/migrations/009_stg_schemas.sql | 4 ++ sql/seed/work_types.sql | 22 ++++++ 13 files changed, 398 insertions(+) create mode 100644 .env.example create mode 100644 docker-compose.yml create mode 100644 infra/init-bit-flight-deck-db.sql create mode 100644 sql/migrations/001_schemas.sql create mode 100644 sql/migrations/002_core_employee.sql create mode 100644 sql/migrations/003_core_project.sql create mode 100644 sql/migrations/004_core_work_log.sql create mode 100644 sql/migrations/005_core_task.sql create mode 100644 sql/migrations/006_core_deal.sql create mode 100644 sql/migrations/007_core_identity_map.sql create mode 100644 sql/migrations/008_raw_schemas.sql create mode 100644 sql/migrations/009_stg_schemas.sql create mode 100644 sql/seed/work_types.sql diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..87a215d --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Copy to .env and fill in real values. .env is gitignored. + +# PostgreSQL — workload-проектная БД в pipeline_postgres +PG_USER=bit_flight_deck_user +PG_PASSWORD= +PG_DB=bit_flight_deck +PG_HOST=pipeline_postgres +PG_PORT=5432 + +# BIT.RA HTTP-сервисы (Phase 3 — пишутся в 1С Конфигуратор) +BITRA_BASE_URL=http:////hs/IntegrationAPI/v1 +BITRA_USER=bit_flight_deck_api +BITRA_PASSWORD= + +# EVA Desk +EVA_BASE_URL=https://firstbit.evateam.ru/api/ +EVA_ADMIN_TOKEN= + +# Bitrix24 +BITRIX_WEBHOOK_URL=https://vdst421.1cbit.ru/rest/91// +BITRIX_WEBHOOK_SECRET= + +# Metabase +METABASE_SITE_NAME=bit-flight-deck +METABASE_PORT=3001 + +# NocoDB +NOCODB_PORT=8090 +NOCODB_ADMIN_EMAIL=roachesnokov@gmail.com +NOCODB_ADMIN_PASSWORD= diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..204b615 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +services: + metabase: + image: metabase/metabase:latest + container_name: bfd_metabase + restart: unless-stopped + ports: + - "${METABASE_PORT:-3001}:3000" + environment: + MB_DB_TYPE: postgres + MB_DB_DBNAME: ${PG_DB} + MB_DB_PORT: 5432 + MB_DB_USER: ${PG_USER} + MB_DB_PASS: ${PG_PASSWORD} + MB_DB_HOST: ${PG_HOST} + MB_SITE_NAME: ${METABASE_SITE_NAME:-bit-flight-deck} + networks: + - pipeline_net + + nocodb: + image: nocodb/nocodb:latest + container_name: bfd_nocodb + restart: unless-stopped + ports: + - "${NOCODB_PORT:-8090}:8080" + environment: + NC_DB: "pg://${PG_HOST}:5432?u=${PG_USER}&p=${PG_PASSWORD}&d=${PG_DB}" + NC_ADMIN_EMAIL: ${NOCODB_ADMIN_EMAIL} + NC_ADMIN_PASSWORD: ${NOCODB_ADMIN_PASSWORD} + networks: + - pipeline_net + +networks: + pipeline_net: + external: true diff --git a/infra/init-bit-flight-deck-db.sql b/infra/init-bit-flight-deck-db.sql new file mode 100644 index 0000000..1e202f2 --- /dev/null +++ b/infra/init-bit-flight-deck-db.sql @@ -0,0 +1,15 @@ +-- Runs ONCE on pipeline_postgres as superuser to create the bit_flight_deck DB and user. +-- +-- Usage: +-- docker exec -e PGPASSWORD= -i pipeline_postgres \ +-- psql -h localhost -U pipeline_user -d postgres \ +-- -v password=$(grep PG_PASSWORD ../.env | cut -d= -f2) \ +-- -f infra/init-bit-flight-deck-db.sql +-- +-- Or inline (less safe, password visible in shell history): +-- psql ... -v password='your-password' -f infra/init-bit-flight-deck-db.sql + +CREATE DATABASE bit_flight_deck; +CREATE USER bit_flight_deck_user WITH PASSWORD :'password'; +GRANT ALL PRIVILEGES ON DATABASE bit_flight_deck TO bit_flight_deck_user; +ALTER DATABASE bit_flight_deck OWNER TO bit_flight_deck_user; diff --git a/sql/migrations/001_schemas.sql b/sql/migrations/001_schemas.sql new file mode 100644 index 0000000..1674a66 --- /dev/null +++ b/sql/migrations/001_schemas.sql @@ -0,0 +1,15 @@ +-- 001_schemas.sql — базовые схемы для слоёв DWH +CREATE SCHEMA IF NOT EXISTS raw_bitra; +CREATE SCHEMA IF NOT EXISTS raw_eva; +CREATE SCHEMA IF NOT EXISTS raw_bitrix; +CREATE SCHEMA IF NOT EXISTS stg_bitra; +CREATE SCHEMA IF NOT EXISTS stg_eva; +CREATE SCHEMA IF NOT EXISTS stg_bitrix; +CREATE SCHEMA IF NOT EXISTS core; +CREATE SCHEMA IF NOT EXISTS mart; + +-- Журнал применённых миграций +CREATE TABLE IF NOT EXISTS public.migrations ( + filename text PRIMARY KEY, + applied_at timestamptz DEFAULT now() +); diff --git a/sql/migrations/002_core_employee.sql b/sql/migrations/002_core_employee.sql new file mode 100644 index 0000000..4a057c0 --- /dev/null +++ b/sql/migrations/002_core_employee.sql @@ -0,0 +1,48 @@ +-- 002_core_employee.sql — core.office, core.department, core.employee, history + +CREATE TABLE core.office ( + id bigserial PRIMARY KEY, + name text NOT NULL, + bitra_id text UNIQUE +); + +CREATE TABLE core.department ( + id bigserial PRIMARY KEY, + name text NOT NULL, + bitra_id text UNIQUE, + bitrix_id bigint UNIQUE, + parent_id bigint REFERENCES core.department, + source text NOT NULL DEFAULT 'bitra' -- bitra | bitrix +); + +CREATE TABLE core.employee ( + id bigserial PRIMARY KEY, + email text UNIQUE NOT NULL, + full_name text, + first_name text, + last_name text, + bitra_user_id text UNIQUE, + eva_person_id text UNIQUE, + bitrix_user_id bigint UNIQUE, + rate decimal(10,2), + office_id bigint REFERENCES core.office, + department_id bigint REFERENCES core.department, + is_active boolean DEFAULT true, + is_target_for_mvp1 boolean DEFAULT false, + last_synced timestamptz +); + +CREATE INDEX idx_employee_email ON core.employee (lower(email)); +CREATE INDEX idx_employee_bitra_id ON core.employee (bitra_user_id); +CREATE INDEX idx_employee_eva_id ON core.employee (eva_person_id); +CREATE INDEX idx_employee_bitrix_id ON core.employee (bitrix_user_id); +CREATE INDEX idx_employee_target_mvp1 ON core.employee (is_target_for_mvp1) WHERE is_target_for_mvp1 = true; + +CREATE TABLE core.department_history ( + employee_id bigint REFERENCES core.employee, + valid_from date NOT NULL, + valid_to date, + department_id bigint REFERENCES core.department, + source text NOT NULL DEFAULT 'bitra', + PRIMARY KEY (employee_id, valid_from) +); diff --git a/sql/migrations/003_core_project.sql b/sql/migrations/003_core_project.sql new file mode 100644 index 0000000..d9e3d58 --- /dev/null +++ b/sql/migrations/003_core_project.sql @@ -0,0 +1,37 @@ +-- 003_core_project.sql — core.project + core.stage + +CREATE TABLE core.project ( + id bigserial PRIMARY KEY, + name text NOT NULL, + bitra_id text UNIQUE, + eva_id text UNIQUE, + bitra_code text, + eva_code text, + is_sd boolean DEFAULT false, -- SD-проект (из явного списка) + status text, + cache_status_type text, -- из EVA: OPEN/IN_PROGRESS/IN_REVIEW/CLOSED + project_manager_id bigint REFERENCES core.employee, + bitrix_company_id bigint, + bitra_client_id text, + bitra_client_name text, -- для отображения + deadline date, + budget decimal(15,2), + last_synced timestamptz +); + +CREATE INDEX idx_project_bitra ON core.project (bitra_id); +CREATE INDEX idx_project_eva ON core.project (eva_id); +CREATE INDEX idx_project_sd ON core.project (is_sd) WHERE is_sd = true; + +CREATE TABLE core.stage ( + id bigserial PRIMARY KEY, + name text NOT NULL, + bitra_id text UNIQUE, + project_id bigint REFERENCES core.project, + plan_start_date date, + plan_end_date date, + is_completed boolean DEFAULT false, + is_acted boolean DEFAULT false +); + +CREATE INDEX idx_stage_project ON core.stage (project_id); diff --git a/sql/migrations/004_core_work_log.sql b/sql/migrations/004_core_work_log.sql new file mode 100644 index 0000000..96fe015 --- /dev/null +++ b/sql/migrations/004_core_work_log.sql @@ -0,0 +1,29 @@ +-- 004_core_work_log.sql — core.work_type + core.work_log + +CREATE TABLE core.work_type ( + code text PRIMARY KEY, + label text NOT NULL, + category text NOT NULL CHECK (category IN ('commercial','presale','internal','free','ignored')), + is_billable boolean NOT NULL DEFAULT false +); + +CREATE TABLE core.work_log ( + id bigserial PRIMARY KEY, + employee_id bigint NOT NULL REFERENCES core.employee, + work_date date NOT NULL, + work_type_code text NOT NULL REFERENCES core.work_type, + project_id bigint REFERENCES core.project, + stage_id bigint REFERENCES core.stage, + bitra_client_id text, + bitra_client_name text, + hours decimal(10,2) NOT NULL, + description text, + bitra_doc_id text NOT NULL, + bitra_row_index int NOT NULL, + UNIQUE (bitra_doc_id, bitra_row_index) +); + +CREATE INDEX idx_work_log_employee_date ON core.work_log (employee_id, work_date DESC); +CREATE INDEX idx_work_log_project ON core.work_log (project_id); +CREATE INDEX idx_work_log_work_type ON core.work_log (work_type_code); +CREATE INDEX idx_work_log_date ON core.work_log (work_date DESC); diff --git a/sql/migrations/005_core_task.sql b/sql/migrations/005_core_task.sql new file mode 100644 index 0000000..a85acb7 --- /dev/null +++ b/sql/migrations/005_core_task.sql @@ -0,0 +1,23 @@ +-- 005_core_task.sql — core.task (EVA-задачи) + +CREATE TABLE core.task ( + id bigserial PRIMARY KEY, + eva_id text UNIQUE NOT NULL, + code text, + name text, + project_id bigint REFERENCES core.project, + responsible_id bigint REFERENCES core.employee, + cache_status_type text NOT NULL CHECK (cache_status_type IN ('OPEN','IN_PROGRESS','IN_REVIEW','CLOSED')), + eva_status_id text, + eva_status_name text, + cmf_created_at timestamptz, + cmf_modified_at timestamptz, + status_in_progress_start timestamptz, + deadline timestamptz, + last_synced timestamptz +); + +CREATE INDEX idx_task_responsible ON core.task (responsible_id); +CREATE INDEX idx_task_status ON core.task (cache_status_type); +CREATE INDEX idx_task_project ON core.task (project_id); +CREATE INDEX idx_task_modified ON core.task (cmf_modified_at DESC); diff --git a/sql/migrations/006_core_deal.sql b/sql/migrations/006_core_deal.sql new file mode 100644 index 0000000..119924c --- /dev/null +++ b/sql/migrations/006_core_deal.sql @@ -0,0 +1,33 @@ +-- 006_core_deal.sql — core.deal + core.deal_team_member (Битрикс CAT=16) + +CREATE TABLE core.deal ( + id bigserial PRIMARY KEY, + bitrix_id bigint UNIQUE NOT NULL, + title text, + category_id int, + stage_id text, + stage_semantic_id char(1), + opportunity decimal(15,2), + begindate date, + closedate date, + assigned_to_id bigint REFERENCES core.employee, + bitrix_company_id bigint, + bitrix_company_name text, + project_manager_id bigint REFERENCES core.employee, + is_in_forecast boolean DEFAULT false, + last_synced timestamptz +); + +CREATE INDEX idx_deal_stage ON core.deal (stage_id); +CREATE INDEX idx_deal_forecast ON core.deal (is_in_forecast) WHERE is_in_forecast = true; +CREATE INDEX idx_deal_category ON core.deal (category_id); + +CREATE TABLE core.deal_team_member ( + deal_id bigint REFERENCES core.deal ON DELETE CASCADE, + employee_id bigint REFERENCES core.employee, + weight decimal(5,2) NOT NULL DEFAULT 1.0, + is_manual_override boolean DEFAULT false, + PRIMARY KEY (deal_id, employee_id) +); + +CREATE INDEX idx_deal_team_employee ON core.deal_team_member (employee_id); diff --git a/sql/migrations/007_core_identity_map.sql b/sql/migrations/007_core_identity_map.sql new file mode 100644 index 0000000..03d79b8 --- /dev/null +++ b/sql/migrations/007_core_identity_map.sql @@ -0,0 +1,19 @@ +-- 007_core_identity_map.sql — таблица соответствий ID между системами + +CREATE TABLE core.identity_map ( + id bigserial PRIMARY KEY, + entity_type text NOT NULL CHECK (entity_type IN ('employee','project','client')), + core_id bigint, + bitra_id text, + eva_id text, + bitrix_id bigint, + confidence text NOT NULL CHECK (confidence IN ('auto','confirmed','manual')), + match_key text, + confirmed_by text, + confirmed_at timestamptz, + created_at timestamptz DEFAULT now(), + UNIQUE (entity_type, bitra_id, eva_id, bitrix_id) +); + +CREATE INDEX idx_identity_map_core ON core.identity_map (entity_type, core_id); +CREATE INDEX idx_identity_map_confidence ON core.identity_map (confidence) WHERE confidence = 'manual'; diff --git a/sql/migrations/008_raw_schemas.sql b/sql/migrations/008_raw_schemas.sql new file mode 100644 index 0000000..bbd1866 --- /dev/null +++ b/sql/migrations/008_raw_schemas.sql @@ -0,0 +1,89 @@ +-- 008_raw_schemas.sql — JSONB-снимки источников + +-- BIT.RA raw +CREATE TABLE raw_bitra.employees ( + bitra_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitra.works ( + bitra_doc_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitra.projects ( + bitra_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitra.dictionaries ( + kind text NOT NULL, + bitra_id text NOT NULL, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now(), + PRIMARY KEY (kind, bitra_id) +); +CREATE TABLE raw_bitra.work_types ( + bitra_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitra.dept_history ( + bitra_employee_id text NOT NULL, + period date NOT NULL, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now(), + PRIMARY KEY (bitra_employee_id, period) +); + +-- EVA raw +CREATE TABLE raw_eva.persons ( + eva_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_eva.projects ( + eva_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_eva.tasks ( + eva_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_eva.status_history ( + eva_id text PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); + +-- Bitrix raw +CREATE TABLE raw_bitrix.deals ( + bitrix_id bigint PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitrix.users ( + bitrix_id bigint PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); +CREATE TABLE raw_bitrix.departments ( + bitrix_id bigint PRIMARY KEY, + payload jsonb NOT NULL, + synced_at timestamptz DEFAULT now() +); + +-- Журнал синхронизаций (общий для всех источников) +CREATE TABLE public.sync_log ( + id bigserial PRIMARY KEY, + source text NOT NULL, + entity text NOT NULL, + last_sync_ts timestamptz, + records_count int, + status text, + error_message text, + synced_at timestamptz DEFAULT now() +); +CREATE INDEX idx_sync_log_source_entity ON public.sync_log (source, entity, synced_at DESC); diff --git a/sql/migrations/009_stg_schemas.sql b/sql/migrations/009_stg_schemas.sql new file mode 100644 index 0000000..a13ebd7 --- /dev/null +++ b/sql/migrations/009_stg_schemas.sql @@ -0,0 +1,4 @@ +-- 009_stg_schemas.sql +-- stg-слой реализован как VIEW в sql/views/stg_*.sql. +-- Эта миграция — placeholder для трекинга порядка применения. +SELECT 1; diff --git a/sql/seed/work_types.sql b/sql/seed/work_types.sql new file mode 100644 index 0000000..1ce4e7e --- /dev/null +++ b/sql/seed/work_types.sql @@ -0,0 +1,22 @@ +-- work_types.sql — 14 видов работ Enum.ВидыРабот из BIT.RA +-- См. memory: reference-bitra-work-types + +INSERT INTO core.work_type (code, label, category, is_billable) VALUES + ('ЛУРВ', 'ЛУРВ (платно)', 'commercial', true), + ('ЛТ', 'ЛТ (платно)', 'commercial', true), + ('Демо', 'Демо (Пресейл)', 'presale', false), + ('ИТС', 'ИТС (договор)', 'commercial', true), + ('ИТСПлатныеРаботы', 'ИТС (доп. услуги)', 'commercial', true), + ('Сертификация', 'Сертификация', 'internal', false), + ('Внутреннее', 'Обучение (кроме сертификации)', 'internal', false), + ('НеОпл', 'Бесплатные часы в счёт ПП', 'free', false), + ('Установка', 'Установка в счёт ПП', 'free', false), + ('Гарантия', 'Гарантия (бесплатно)', 'free', false), + ('Управленка', 'Работа руководителя', 'internal', false), + ('ВнутренниеРаботы', 'Внутренние работы', 'internal', false), + ('Коробка', 'Коробка (рудимент)', 'ignored', false), + ('Отложено', 'Отложено (рудимент)', 'ignored', false) +ON CONFLICT (code) DO UPDATE SET + label = EXCLUDED.label, + category = EXCLUDED.category, + is_billable = EXCLUDED.is_billable;