"""
BA3 Text as Data: Korean preprocessing for Orange (Mac and Windows)

Load this file in Orange's Python Script widget (More > Import Script from File) and click Run.
Input:  a Corpus (for example presidential-speeches-100.csv).
Output: the same table plus a new text column called 'processed_text'.

The script runs the three preprocessing steps from the lecture:
  STEP 1 Clean: remove links, punctuation and symbols.
  STEP 2 Tokenize and tag (POS): Kiwi splits each eojeol (a space-separated
         unit) into morphemes, gives each a part-of-speech tag, and returns
         its base form.
  STEP 3 Filter: keep the chosen POS tags; drop stopwords, one-syllable
         tokens and numbers.
Copy the SETTINGS block into your Demo 2 README.md every time you run it.
"""

import os
import sys
import re
import site
import importlib
import subprocess

import pandas as pd
from Orange.data import Domain, StringVariable

# ===== SETTINGS (copy this block into your Demo 2 README.md) =====

# The column that holds the full speech text.
TEXT_COLUMN = 'speech_text'

# STEP 3: POS tags to keep. Everything else (particles, endings, ...) is dropped.
POS_TAGS = [
    'NNG',  # general noun (일반명사), e.g. 경제, 국민
    'NNP',  # proper noun (고유명사), e.g. 한국, 서울
    # To also keep verb (VV, 동사) and adjective (VA, 형용사) stems,
    # delete the # before 'VV' and 'VA':
    # 'VV',
    # 'VA',
]

REMOVE_NUMBERS = True   # STEP 3: drop tokens made only of digits, e.g. 2026
MIN_TOKEN_LENGTH = 2    # STEP 3: drop one-syllable tokens such as 것 or 수

# STEP 3: Kiwi splits a negating prefix off its noun, 불안정 -> 불 + 안정, and the filter above
# drops the prefix, so 'instability' would count as 안정 'stability'. Set it to True to glue it back on.
JOIN_PREFIXES = False
NEGATING_PREFIXES = {'불', '비', '미', '무', '반', '몰', '부'}

# STEP 3: a short safety-net stopword list (the main list is stopwords-ko.txt in Preprocess Text).
STOPWORDS = {
    '있다', '없다', '되다', '하다', '그', '저', '이', '것', '등', '및',
    '수', '때', '년', '월', '일', '더', '또', '즉', '통해', '위해'
}

# ===== SETUP: INSTALL KIWI IF NEEDED (first run only) =====
# The first run downloads Kiwi from the internet. This takes a minute or two.

def pip_install(extra_args):
    # Use the same Python that runs Orange. On Windows Orange runs as pythonw.exe,
    # which has no console, so run pip with python.exe instead (as Orange's add-on installer does).
    exe = sys.executable
    if os.name == 'nt' and os.path.basename(exe).lower() == 'pythonw.exe':
        exe = os.path.join(os.path.dirname(exe), 'python.exe')
    env = dict(os.environ, PYTHONIOENCODING='utf-8', PYTHONUTF8='1')
    result = subprocess.run(
        [exe, '-m', 'pip', 'install'] + extra_args + ['kiwipiepy'],
        env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        text=True, encoding='utf-8', errors='replace',
        creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0),
    )
    if result.returncode != 0:
        print('\n'.join(result.stdout.splitlines()[-5:]))
    return result.returncode == 0

def install_kiwi():
    print('Kiwi (kiwipiepy) is not installed yet. Installing it now; '
          'this needs an internet connection and takes a minute or two.')
    if not pip_install([]):
        print('Standard install failed. Trying a user-only install instead.')
        if pip_install(['--user']):
            user_site = site.getusersitepackages()
            if user_site not in sys.path:
                sys.path.append(user_site)
    importlib.invalidate_caches()

try:
    import kiwipiepy
except ImportError:
    install_kiwi()
    try:
        import kiwipiepy
    except ImportError:
        message = ('Kiwi could not be installed automatically. Check your internet '
                   'connection, close and reopen Orange, and run the script again. '
                   'If it still fails, ask the instructor and include this message.')
        print(message)
        raise RuntimeError(message)

from kiwipiepy import Kiwi

kiwi = Kiwi()  # the Korean morpheme analyzer used in STEP 2

# ===== STEP 1 Clean =====

def clean_text(text):
    """Remove links, email addresses, @mentions and punctuation before Kiwi runs."""
    if text is None or pd.isna(text):
        return ''
    text = str(text)
    text = re.sub(r'http[s]?://\S+', '', text)   # web links
    text = re.sub(r'\S+@\S+', '', text)          # email addresses
    text = re.sub(r'@\w+', '', text)             # @mentions
    # Punctuation and symbols become spaces; Hangul, letters and digits stay.
    text = re.sub(r'[^\w\s\u3131-\u3163\uac00-\ud7a3\u1100-\u11ff]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()     # collapse repeated spaces
    return text

# ===== STEP 2 Tokenize and tag (POS), then STEP 3 Filter =====

def preprocess(text):
    """Return the kept base forms of one document, separated by spaces."""
    text = clean_text(text)  # STEP 1
    if not text:
        return ''
    kept = []
    # STEP 2: Kiwi splits each eojeol into morphemes, e.g. 국민의 -> 국민 (NNG) + 의 (particle).
    # token.form is the base form; token.tag is its POS tag.
    prefix, prefix_end = '', -1
    for token in kiwi.tokenize(text):
        word = token.form
        if JOIN_PREFIXES and token.tag == 'XPN' and word in NEGATING_PREFIXES:
            prefix, prefix_end = word, token.start + token.len
            continue
        if prefix and token.start == prefix_end:
            word = prefix + word  # 불 + 안정 -> 불안정
        prefix = ''
        # STEP 3: keep the token only if it passes every check below.
        # Kiwi marks some verb tags with a suffix (VV-R, VA-I); compare the base tag.
        if token.tag.split('-')[0] not in POS_TAGS:
            continue
        if word in STOPWORDS:
            continue
        if len(word) < MIN_TOKEN_LENGTH:
            continue
        if REMOVE_NUMBERS and word.isdigit():
            continue
        kept.append(word)
    return ' '.join(kept)

# ===== RUN ON THE CORPUS (all three steps, every document) =====

if in_data is None:
    raise RuntimeError('No data. Connect a Corpus widget to this Python Script widget first.')

column_names = [var.name for var in in_data.domain.variables + in_data.domain.metas]
if TEXT_COLUMN in column_names:
    texts = list(in_data.get_column(TEXT_COLUMN))
elif hasattr(in_data, 'documents'):
    print(f"Column '{TEXT_COLUMN}' not found; using the Corpus text feature instead.")
    texts = list(in_data.documents)
else:
    raise RuntimeError(f"Column '{TEXT_COLUMN}' not found. Change TEXT_COLUMN "
                       f"to one of: {', '.join(column_names)}")

processed = [preprocess(text) for text in texts]

# ===== OUTPUT: add 'processed_text' as a new text column =====

new_var = StringVariable('processed_text')
new_domain = Domain(
    in_data.domain.attributes,
    in_data.domain.class_vars,
    in_data.domain.metas + (new_var,)
)
out_data = in_data.transform(new_domain)  # keeps the Corpus type
with out_data.unlocked(out_data.metas):
    out_data.metas[:, -1] = processed

token_counts = [len(doc.split()) for doc in processed]
mean_tokens = sum(token_counts) / len(token_counts) if token_counts else 0
print(f'Documents processed: {len(processed)}')
print(f'Mean tokens kept per document: {mean_tokens:.1f}')
print("Next: in a Corpus widget, set 'processed_text' as the text feature.")