feat(phase-0,phase-2): bootstrap DB schemas + Metabase/NocoDB compose + work_types seed

.env.example

@@ -0,0 +1,30 @@
+# Copy to .env and fill in real values. .env is gitignored.
+
+# PostgreSQL — workload-проектная БД в pipeline_postgres
+PG_USER=bit_flight_deck_user
+PG_PASSWORD=<set after init-bit-flight-deck-db.sql>
+PG_DB=bit_flight_deck
+PG_HOST=pipeline_postgres
+PG_PORT=5432
+
+# BIT.RA HTTP-сервисы (Phase 3 — пишутся в 1С Конфигуратор)
+BITRA_BASE_URL=http://<host>/<dbname>/hs/IntegrationAPI/v1
+BITRA_USER=bit_flight_deck_api
+BITRA_PASSWORD=<set in BIT.RA admin panel>
+
+# EVA Desk
+EVA_BASE_URL=https://firstbit.evateam.ru/api/
+EVA_ADMIN_TOKEN=<global admin token from EVA admin panel>
+
+# Bitrix24
+BITRIX_WEBHOOK_URL=https://vdst421.1cbit.ru/rest/91/<webhook-token>/
+BITRIX_WEBHOOK_SECRET=<random; placed in the n8n webhook path so only Bitrix knows it>
+
+# Metabase
+METABASE_SITE_NAME=bit-flight-deck
+METABASE_PORT=3001
+
+# NocoDB
+NOCODB_PORT=8090
+NOCODB_ADMIN_EMAIL=roachesnokov@gmail.com
+NOCODB_ADMIN_PASSWORD=<set>

docker-compose.yml

@@ -0,0 +1,34 @@
+services:
+  metabase:
+    image: metabase/metabase:latest
+    container_name: bfd_metabase
+    restart: unless-stopped
+    ports:
+      - "${METABASE_PORT:-3001}:3000"
+    environment:
+      MB_DB_TYPE: postgres
+      MB_DB_DBNAME: ${PG_DB}
+      MB_DB_PORT: 5432
+      MB_DB_USER: ${PG_USER}
+      MB_DB_PASS: ${PG_PASSWORD}
+      MB_DB_HOST: ${PG_HOST}
+      MB_SITE_NAME: ${METABASE_SITE_NAME:-bit-flight-deck}
+    networks:
+      - pipeline_net
+
+  nocodb:
+    image: nocodb/nocodb:latest
+    container_name: bfd_nocodb
+    restart: unless-stopped
+    ports:
+      - "${NOCODB_PORT:-8090}:8080"
+    environment:
+      NC_DB: "pg://${PG_HOST}:5432?u=${PG_USER}&p=${PG_PASSWORD}&d=${PG_DB}"
+      NC_ADMIN_EMAIL: ${NOCODB_ADMIN_EMAIL}
+      NC_ADMIN_PASSWORD: ${NOCODB_ADMIN_PASSWORD}
+    networks:
+      - pipeline_net
+
+networks:
+  pipeline_net:
+    external: true

infra/init-bit-flight-deck-db.sql

@@ -0,0 +1,15 @@
+-- Runs ONCE on pipeline_postgres as superuser to create the bit_flight_deck DB and user.
+--
+-- Usage:
+--   docker exec -e PGPASSWORD=<superuser-password> -i pipeline_postgres \
+--     psql -h localhost -U pipeline_user -d postgres \
+--     -v password=$(grep PG_PASSWORD ../.env | cut -d= -f2) \
+--     -f infra/init-bit-flight-deck-db.sql
+--
+-- Or inline (less safe, password visible in shell history):
+--   psql ... -v password='your-password' -f infra/init-bit-flight-deck-db.sql
+
+CREATE DATABASE bit_flight_deck;
+CREATE USER bit_flight_deck_user WITH PASSWORD :'password';
+GRANT ALL PRIVILEGES ON DATABASE bit_flight_deck TO bit_flight_deck_user;
+ALTER DATABASE bit_flight_deck OWNER TO bit_flight_deck_user;

sql/migrations/001_schemas.sql

@@ -0,0 +1,15 @@
+-- 001_schemas.sql — базовые схемы для слоёв DWH
+CREATE SCHEMA IF NOT EXISTS raw_bitra;
+CREATE SCHEMA IF NOT EXISTS raw_eva;
+CREATE SCHEMA IF NOT EXISTS raw_bitrix;
+CREATE SCHEMA IF NOT EXISTS stg_bitra;
+CREATE SCHEMA IF NOT EXISTS stg_eva;
+CREATE SCHEMA IF NOT EXISTS stg_bitrix;
+CREATE SCHEMA IF NOT EXISTS core;
+CREATE SCHEMA IF NOT EXISTS mart;
+
+-- Журнал применённых миграций
+CREATE TABLE IF NOT EXISTS public.migrations (
+    filename text PRIMARY KEY,
+    applied_at timestamptz DEFAULT now()
+);

sql/migrations/002_core_employee.sql

@@ -0,0 +1,48 @@
+-- 002_core_employee.sql — core.office, core.department, core.employee, history
+
+CREATE TABLE core.office (
+    id bigserial PRIMARY KEY,
+    name text NOT NULL,
+    bitra_id text UNIQUE
+);
+
+CREATE TABLE core.department (
+    id bigserial PRIMARY KEY,
+    name text NOT NULL,
+    bitra_id text UNIQUE,
+    bitrix_id bigint UNIQUE,
+    parent_id bigint REFERENCES core.department,
+    source text NOT NULL DEFAULT 'bitra'   -- bitra | bitrix
+);
+
+CREATE TABLE core.employee (
+    id bigserial PRIMARY KEY,
+    email text UNIQUE NOT NULL,
+    full_name text,
+    first_name text,
+    last_name text,
+    bitra_user_id text UNIQUE,
+    eva_person_id text UNIQUE,
+    bitrix_user_id bigint UNIQUE,
+    rate decimal(10,2),
+    office_id bigint REFERENCES core.office,
+    department_id bigint REFERENCES core.department,
+    is_active boolean DEFAULT true,
+    is_target_for_mvp1 boolean DEFAULT false,
+    last_synced timestamptz
+);
+
+CREATE INDEX idx_employee_email ON core.employee (lower(email));
+CREATE INDEX idx_employee_bitra_id ON core.employee (bitra_user_id);
+CREATE INDEX idx_employee_eva_id ON core.employee (eva_person_id);
+CREATE INDEX idx_employee_bitrix_id ON core.employee (bitrix_user_id);
+CREATE INDEX idx_employee_target_mvp1 ON core.employee (is_target_for_mvp1) WHERE is_target_for_mvp1 = true;
+
+CREATE TABLE core.department_history (
+    employee_id bigint REFERENCES core.employee,
+    valid_from date NOT NULL,
+    valid_to date,
+    department_id bigint REFERENCES core.department,
+    source text NOT NULL DEFAULT 'bitra',
+    PRIMARY KEY (employee_id, valid_from)
+);

sql/migrations/003_core_project.sql

@@ -0,0 +1,37 @@
+-- 003_core_project.sql — core.project + core.stage
+
+CREATE TABLE core.project (
+    id bigserial PRIMARY KEY,
+    name text NOT NULL,
+    bitra_id text UNIQUE,
+    eva_id text UNIQUE,
+    bitra_code text,
+    eva_code text,
+    is_sd boolean DEFAULT false,           -- SD-проект (из явного списка)
+    status text,
+    cache_status_type text,                 -- из EVA: OPEN/IN_PROGRESS/IN_REVIEW/CLOSED
+    project_manager_id bigint REFERENCES core.employee,
+    bitrix_company_id bigint,
+    bitra_client_id text,
+    bitra_client_name text,                 -- для отображения
+    deadline date,
+    budget decimal(15,2),
+    last_synced timestamptz
+);
+
+CREATE INDEX idx_project_bitra ON core.project (bitra_id);
+CREATE INDEX idx_project_eva ON core.project (eva_id);
+CREATE INDEX idx_project_sd ON core.project (is_sd) WHERE is_sd = true;
+
+CREATE TABLE core.stage (
+    id bigserial PRIMARY KEY,
+    name text NOT NULL,
+    bitra_id text UNIQUE,
+    project_id bigint REFERENCES core.project,
+    plan_start_date date,
+    plan_end_date date,
+    is_completed boolean DEFAULT false,
+    is_acted boolean DEFAULT false
+);
+
+CREATE INDEX idx_stage_project ON core.stage (project_id);

sql/migrations/004_core_work_log.sql

@@ -0,0 +1,29 @@
+-- 004_core_work_log.sql — core.work_type + core.work_log
+
+CREATE TABLE core.work_type (
+    code text PRIMARY KEY,
+    label text NOT NULL,
+    category text NOT NULL CHECK (category IN ('commercial','presale','internal','free','ignored')),
+    is_billable boolean NOT NULL DEFAULT false
+);
+
+CREATE TABLE core.work_log (
+    id bigserial PRIMARY KEY,
+    employee_id bigint NOT NULL REFERENCES core.employee,
+    work_date date NOT NULL,
+    work_type_code text NOT NULL REFERENCES core.work_type,
+    project_id bigint REFERENCES core.project,
+    stage_id bigint REFERENCES core.stage,
+    bitra_client_id text,
+    bitra_client_name text,
+    hours decimal(10,2) NOT NULL,
+    description text,
+    bitra_doc_id text NOT NULL,
+    bitra_row_index int NOT NULL,
+    UNIQUE (bitra_doc_id, bitra_row_index)
+);
+
+CREATE INDEX idx_work_log_employee_date ON core.work_log (employee_id, work_date DESC);
+CREATE INDEX idx_work_log_project ON core.work_log (project_id);
+CREATE INDEX idx_work_log_work_type ON core.work_log (work_type_code);
+CREATE INDEX idx_work_log_date ON core.work_log (work_date DESC);

sql/migrations/005_core_task.sql

@@ -0,0 +1,23 @@
+-- 005_core_task.sql — core.task (EVA-задачи)
+
+CREATE TABLE core.task (
+    id bigserial PRIMARY KEY,
+    eva_id text UNIQUE NOT NULL,
+    code text,
+    name text,
+    project_id bigint REFERENCES core.project,
+    responsible_id bigint REFERENCES core.employee,
+    cache_status_type text NOT NULL CHECK (cache_status_type IN ('OPEN','IN_PROGRESS','IN_REVIEW','CLOSED')),
+    eva_status_id text,
+    eva_status_name text,
+    cmf_created_at timestamptz,
+    cmf_modified_at timestamptz,
+    status_in_progress_start timestamptz,
+    deadline timestamptz,
+    last_synced timestamptz
+);
+
+CREATE INDEX idx_task_responsible ON core.task (responsible_id);
+CREATE INDEX idx_task_status ON core.task (cache_status_type);
+CREATE INDEX idx_task_project ON core.task (project_id);
+CREATE INDEX idx_task_modified ON core.task (cmf_modified_at DESC);

sql/migrations/006_core_deal.sql

@@ -0,0 +1,33 @@
+-- 006_core_deal.sql — core.deal + core.deal_team_member (Битрикс CAT=16)
+
+CREATE TABLE core.deal (
+    id bigserial PRIMARY KEY,
+    bitrix_id bigint UNIQUE NOT NULL,
+    title text,
+    category_id int,
+    stage_id text,
+    stage_semantic_id char(1),
+    opportunity decimal(15,2),
+    begindate date,
+    closedate date,
+    assigned_to_id bigint REFERENCES core.employee,
+    bitrix_company_id bigint,
+    bitrix_company_name text,
+    project_manager_id bigint REFERENCES core.employee,
+    is_in_forecast boolean DEFAULT false,
+    last_synced timestamptz
+);
+
+CREATE INDEX idx_deal_stage ON core.deal (stage_id);
+CREATE INDEX idx_deal_forecast ON core.deal (is_in_forecast) WHERE is_in_forecast = true;
+CREATE INDEX idx_deal_category ON core.deal (category_id);
+
+CREATE TABLE core.deal_team_member (
+    deal_id bigint REFERENCES core.deal ON DELETE CASCADE,
+    employee_id bigint REFERENCES core.employee,
+    weight decimal(5,2) NOT NULL DEFAULT 1.0,
+    is_manual_override boolean DEFAULT false,
+    PRIMARY KEY (deal_id, employee_id)
+);
+
+CREATE INDEX idx_deal_team_employee ON core.deal_team_member (employee_id);

sql/migrations/007_core_identity_map.sql

@@ -0,0 +1,19 @@
+-- 007_core_identity_map.sql — таблица соответствий ID между системами
+
+CREATE TABLE core.identity_map (
+    id bigserial PRIMARY KEY,
+    entity_type text NOT NULL CHECK (entity_type IN ('employee','project','client')),
+    core_id bigint,
+    bitra_id text,
+    eva_id text,
+    bitrix_id bigint,
+    confidence text NOT NULL CHECK (confidence IN ('auto','confirmed','manual')),
+    match_key text,
+    confirmed_by text,
+    confirmed_at timestamptz,
+    created_at timestamptz DEFAULT now(),
+    UNIQUE (entity_type, bitra_id, eva_id, bitrix_id)
+);
+
+CREATE INDEX idx_identity_map_core ON core.identity_map (entity_type, core_id);
+CREATE INDEX idx_identity_map_confidence ON core.identity_map (confidence) WHERE confidence = 'manual';

sql/migrations/008_raw_schemas.sql

@@ -0,0 +1,89 @@
+-- 008_raw_schemas.sql — JSONB-снимки источников
+
+-- BIT.RA raw
+CREATE TABLE raw_bitra.employees (
+    bitra_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitra.works (
+    bitra_doc_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitra.projects (
+    bitra_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitra.dictionaries (
+    kind text NOT NULL,
+    bitra_id text NOT NULL,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now(),
+    PRIMARY KEY (kind, bitra_id)
+);
+CREATE TABLE raw_bitra.work_types (
+    bitra_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitra.dept_history (
+    bitra_employee_id text NOT NULL,
+    period date NOT NULL,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now(),
+    PRIMARY KEY (bitra_employee_id, period)
+);
+
+-- EVA raw
+CREATE TABLE raw_eva.persons (
+    eva_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_eva.projects (
+    eva_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_eva.tasks (
+    eva_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_eva.status_history (
+    eva_id text PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+
+-- Bitrix raw
+CREATE TABLE raw_bitrix.deals (
+    bitrix_id bigint PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitrix.users (
+    bitrix_id bigint PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE TABLE raw_bitrix.departments (
+    bitrix_id bigint PRIMARY KEY,
+    payload jsonb NOT NULL,
+    synced_at timestamptz DEFAULT now()
+);
+
+-- Журнал синхронизаций (общий для всех источников)
+CREATE TABLE public.sync_log (
+    id bigserial PRIMARY KEY,
+    source text NOT NULL,
+    entity text NOT NULL,
+    last_sync_ts timestamptz,
+    records_count int,
+    status text,
+    error_message text,
+    synced_at timestamptz DEFAULT now()
+);
+CREATE INDEX idx_sync_log_source_entity ON public.sync_log (source, entity, synced_at DESC);

sql/migrations/009_stg_schemas.sql

@@ -0,0 +1,4 @@
+-- 009_stg_schemas.sql
+-- stg-слой реализован как VIEW в sql/views/stg_*.sql.
+-- Эта миграция — placeholder для трекинга порядка применения.
+SELECT 1;

sql/seed/work_types.sql

@@ -0,0 +1,22 @@
+-- work_types.sql — 14 видов работ Enum.ВидыРабот из BIT.RA
+-- См. memory: reference-bitra-work-types
+
+INSERT INTO core.work_type (code, label, category, is_billable) VALUES
+    ('ЛУРВ',              'ЛУРВ (платно)',                  'commercial', true),
+    ('ЛТ',                'ЛТ (платно)',                    'commercial', true),
+    ('Демо',              'Демо (Пресейл)',                 'presale',    false),
+    ('ИТС',               'ИТС (договор)',                  'commercial', true),
+    ('ИТСПлатныеРаботы',  'ИТС (доп. услуги)',              'commercial', true),
+    ('Сертификация',      'Сертификация',                   'internal',   false),
+    ('Внутреннее',        'Обучение (кроме сертификации)',  'internal',   false),
+    ('НеОпл',             'Бесплатные часы в счёт ПП',      'free',       false),
+    ('Установка',         'Установка в счёт ПП',            'free',       false),
+    ('Гарантия',          'Гарантия (бесплатно)',           'free',       false),
+    ('Управленка',        'Работа руководителя',            'internal',   false),
+    ('ВнутренниеРаботы',  'Внутренние работы',              'internal',   false),
+    ('Коробка',           'Коробка (рудимент)',             'ignored',    false),
+    ('Отложено',          'Отложено (рудимент)',            'ignored',    false)
+ON CONFLICT (code) DO UPDATE SET
+    label = EXCLUDED.label,
+    category = EXCLUDED.category,
+    is_billable = EXCLUDED.is_billable;