# -*- coding: utf-8 -*-
# Generated by Django 1.10.4 on 2017-01-05 14:58
from __future__ import unicode_literals

import re

from django.db import migrations

from stemming.porter2 import stem

from pootle.core.delegate import stopwords
from pootle_store.constants import TRANSLATED


def stem_terminology_words(apps, schema_editor):
    Stem = apps.get_model("pootle_word.Stem")
    Unit = apps.get_model("pootle_store.Unit")
    units = Unit.objects.filter(state=TRANSLATED)
    units = (
        units.filter(store__translation_project__project__code="terminology")
        | units.filter(store__name__startswith="pootle-terminology"))
    units = units.values_list("id", "source_f")
    site_stopwords = stopwords.get().words
    stems = {}
    delimiters = re.compile(u"[\W]+", re.U)
    for unit, source in units:
        source_words = set(
            s.strip().lower()
            for s in delimiters.split(source)
            if len(s) > 2)
        for word in source_words:
            if not word:
                continue
            if word in site_stopwords:
                continue
            stemmed = stem(word)
            stems[stemmed] = stems.get(stemmed, set())
            stems[stemmed].add(unit)
    Stem.objects.bulk_create([Stem(root=st) for st in stems], batch_size=1000)
    added = dict(Stem.objects.values_list("root", "id"))
    m2m = Stem.units.through
    m2m_to_add = []
    for stemmed in stems:
        for unit in stems[stemmed]:
            m2m_to_add.append(m2m(stem_id=added[stemmed], unit_id=unit))
    m2m.objects.bulk_create(m2m_to_add, batch_size=1000)


class Migration(migrations.Migration):

    dependencies = [
        ('pootle_word', '0002_set_cs_schema'),
    ]

    operations = [
        migrations.RunPython(stem_terminology_words),
    ]
