TisaneLocal

5.0.2

TisaneLocal.psm1

#Region '.\Private\Save-LocalTisaneSettings.ps1' 0
Function Save-LocalTisaneSettings([String] $settingName, $settingValue){
    [Environment]::SetEnvironmentVariable($settingName, $settingValue, 'User')
}
#EndRegion '.\Private\Save-LocalTisaneSettings.ps1' 4
#Region '.\Public\Create-Tisane.ps1' 0
Function Create-Tisane(){
    $TISANE_PATH = [Environment]::GetEnvironmentVariable('TISANE_PATH', 'User')
    if ([string]::IsNullOrEmpty($TISANE_PATH)){
        $TISANE_PATH = Read-Host -Prompt 'Please enter Tisane Path'
        if ([string]::IsNullOrEmpty($TISANE_PATH)){
            Throw "No path provided"
        }
        Save-LocalTisaneSettings -settingName 'TISANE_PATH' -settingValue $TISANE_PATH
    }

    $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"
    # write-Host $config_path
    [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)

    Add-Type -AssemblyName System.Configuration
    $something = [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
    $something = [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
    ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
    $something = [Configuration.ConfigurationManager]::ConnectionStrings[0].Name

    $something = [Reflection.Assembly]::LoadFrom($TISANE_PATH + "Tisane.Runtime.dll")
    try {
        $tisane = New-Object Tisane.Server
        # $tisane.Parse("en", "this is testing for testing", '{"snippets":true,"topic_standard":"native","sentiment":false,"parses":false,"words":true, "debug":true,"entities":true,"disable_spellcheck":true}')
    }catch{
        Reset-LocalTisaneSettings
        throw $_
    }
    return $tisane
}
# $tisane = Create-Tisane
#EndRegion '.\Public\Create-Tisane.ps1' 32
#Region '.\Public\Delete-RocksDbOldLogs.ps1' 0
function Delete-RocksDbOldLogs{
dir "C:\Tisane" -Recurse | Where-Object { $_.FullName -imatch '^C:\\tisane\\([^\\]+)\\LOG[.]old.+' } | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
}
#EndRegion '.\Public\Delete-RocksDbOldLogs.ps1' 4
#Region '.\Public\Delete-TisaneTranslationDatastores.ps1' 0
function Delete-TisaneTranslationDatastores{
dir "C:/Tisane" -Recurse | Where-Object { $_.FullName -ilike '*famlex' } | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
dir "C:/Tisane" -Recurse | Where-Object { $_.FullName -ilike '*famphrase' } | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
}
#EndRegion '.\Public\Delete-TisaneTranslationDatastores.ps1' 5
#Region '.\Public\Entity-ComparisonTest.ps1' 0
function Entity-ComparisonTest.ps1{
# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  $languageCode = 'en'
  $comparisonResult = $tisane.CompareEntities('ru', 'Сергей Царнаев', 'ru', 'Серёга Царнаев', 'person')
  "Comparison 1:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'Mr Vadim Berman', 'en', 'Prof Vadim Berman', 'person')
  "Comparison 2:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'William Smith', 'ru', 'Вилл Смит', 'person')
  "Comparison 3:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'William Smith', 'en', 'Prof Smith', 'person')
  "Comparison 4:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'William Smith', 'en', 'Prof William Smith', 'person')
  "Comparison 5:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'William Smith', 'en', 'William Smith PhD', 'person')
  "Comparison 6:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'Sir William Smith', 'en', 'William Smith PhD', 'person')
  "Comparison 7:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'Musa Bin Osman', 'en', 'Haji Musa Bin Osman', 'person')
  "Comparison 8:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'Kevin Tan', 'en', 'TAN Kevin', 'person')
  "Comparison 9:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'Sir William Smith', 'en', 'Professor William Smith', 'person')
  "Comparison 10:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('tr', 'Ilhan Kolaç', 'tr', 'Doktor Zahid Kolaç', 'person')
  "Comparison 11:" + $comparisonResult
  $comparisonResult = $tisane.CompareEntities('en', 'William Smith', 'en', 'Will Smith', 'person')
  "Comparison 12:" + $comparisonResult
  $tisane.Parse('en', 'Dr. Juan Q. Xavier de la Vega III', '{}')
  $tisane.Parse('en', 'Juan Q. Xavier Velasquez y Garcia, Jr.', '{}')
  $tisane.Parse('en', 'Jason Alexander', '{}')
  $tisane.Parse('en', 'Doe-Ray, Jonathan "John" A. Harris', '{}')
  $tisane.Parse('en', 'bob v. de la macdole-eisenhower phd', '{}')
  $tisane.Parse('en', 'Jonathan "John" A. Smith', '{}')
  $tisane.Parse('en', 'Robert "Rob" Johnson', '{}')
  $tisane.Parse('en', 'Dr. Juan Ruiz de la Vega III (Doc Vega)', '{}')
  $tisane.Parse('en', 'John A. Doe', '{}')
  $tisane.Parse('en', 'Doe, John A.', '{}')
  $tisane.Parse('en', 'Mr. John Andrew Doe lll', '{}')
  $tisane.Parse('en', 'Sarah Jessica Parker', '{}')
  $tisane.Parse('en', 'Jessie James', '{}')

}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Entity-ComparisonTest.ps1' 60
#Region '.\Public\Excel-BrandTest.ps1' 0
function Excel-BrandTest{
[CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Language(s): ")][String] $language,
     [Parameter(Mandatory = $true, HelpMessage="Spreadsheet filename: ")][String] $filename,
     [Parameter(Mandatory = $true, HelpMessage="1st output column: ")][int] $outputColumnIndex,
     [Parameter(Mandatory = $true, HelpMessage="Start line: ")][int] $startLine,
     [Parameter(Mandatory = $false, HelpMessage="End line: ")][int] $endLine,
     [Parameter(Mandatory = $false, HelpMessage="Sheet index: ")][int] $sheetIndex,
     [Parameter(Mandatory = $false, HelpMessage="Input column: ")][int] $inputColumn
)


# $TISANE_PATH = 'C:/Tisane/TestConsole/'
$ROW_COUNT_IN_SPREADSHEET = 3 #1000

$OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding =
                    New-Object System.Text.UTF8Encoding

$Excel = New-Object -ComObject Excel.Application
$Workbook = $Excel.Workbooks.Open($filename)
if (-not($sheetIndex)) {
  $sheetIndex = 1
}
if (-not($outputColumnIndex)) {
  $outputColumnIndex = 3
}
if (-not($inputColumn)) {
  $inputColumn = 1
}
$sentimentColumnIndex = $outputColumnIndex
$brandColumnIndex = $outputColumnIndex + 1
$topicColumnIndex = $outputColumnIndex + 2
$orgColumnIndex = $outputColumnIndex + 3
$hashtagsColumnIndex = $outputColumnIndex + 4
$aspectsColumnIndex = $outputColumnIndex + 5
$abuseColumnIndex = $outputColumnIndex + 6
$promotionColumnIndex = $outputColumnIndex + 7


$srcSheet = $Workbook.Sheets.Item($sheetIndex)
# $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"

# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path) # assign the configuration file

# # BEGIN fix for Powershell bug: in some cases, the configuration files aren't read properly
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# # END fix for Powershell bug: in some cases, the configuration files aren't read properly

# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll") # load the type

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
Write-Progress -Activity "Loading language model" -Status "Normally takes 30-40 sec"
[Tisane.Server]::ActivateLazyLoading()
$doesntMatter = $tisane.Parse('en', 'Testing', '{}')
if (-not($startLine)) {
  $startLine = 2
}
if (-not($endLine)) {
  $endLine = $startLine + $ROW_COUNT_IN_SPREADSHEET
}

$settings = '{"snippets":true,"topic_standard":"native","sentiment":true,"document_sentiment":true,"parses":false,"domain_match_threshold":1,"words":false,"entities":true,"explain":false,"format":"shortform"}'


For ($i=$startLine; $i -le $endLine; $i++) {
  if ($i % 3000 -eq 0) { $Workbook.Save() }
  $content = $srcSheet.Cells.Item($i,$inputColumn).Text
  if (-not $content) {continue}
  $currentLanguage = $language
  if ($language -eq '*' -or $language -like '*|*') {
    $languageJson = $tisane.DetectLanguage($content, $language, '')
    $languageDetection = $languageJson | ConvertFrom-Json
    if ($languageDetection.languages -and $languageDetection.languages[0].language -and $languageDetection.languages[0].language -ne 'mg' -and $languageDetection.languages[0].language -ne 'lt' -and $languageDetection.languages[0].language -ne 'un' -and $languageDetection.languages[0].language -ne 'cy' -and $languageDetection.languages[0].language -ne 'lt') {
      $currentLanguage = $languageDetection.languages[0].language
    }
  }
  
  $pct = (($i - $startLine) / ($endLine - $startLine)) * 100
  Write-Progress -Activity "[$i] $content" -Status "$pct% complete" -PercentComplete $pct
  $tisaneResult = $tisane.Parse($currentLanguage, $content, $settings)
  #$tisaneResult
  $parsedTisane = ConvertFrom-Json -InputObject $tisaneResult
  #$parsedTisane | select -ExpandProperty sentence_list
  $abuseTags = ''
  $abuseType = ''
  $explanation = ''
  $aspects = ''
  if ($parsedTisane.abuse) {
    $parsedTisane.abuse | Foreach-Object {
      if ($_.type -eq 'bigotry' -or $_.type -eq 'profanity' -or $_.type -eq 'external_contact') {
        #$abuseText = $_.text
        $abuseTags = $abuseTags + $_.tags
        $abuseType = $abuseType + $_.type
        #$explanation = $_.explanation
      }
    }
  }
  
  $topics = ''
  if ($parsedTisane.topics) {
    $parsedTisane.topics | Sort-Object | Foreach-Object {
      if ($_ -eq 'business' -or $_ -eq 'computing' -or $_ -eq 'computer network' -or $_ -eq 'computing' -or $_ -eq 'feeling' -or $_ -eq 'body' -or $_ -eq 'resource' -or $_ -eq 'time' -or $_ -eq 'verbal communication' -or $_ -eq 'written language' -or $_ -eq 'language' -or $_ -eq 'Europe' -or $_ -eq 'France' -or $_ -eq 'Italy' -or $_ -eq 'seeing' -or $_ -eq 'outer space' -or $_ -eq 'astronomy' -or $_ -eq 'question' -or $_ -eq 'weather science' -or $_ -eq 'atmosphere' -or $_ -eq 'ecology' -or $_ -eq 'mathematics' -or $_ -eq 'philosophy' -or $_ -eq 'dishonesty' -or $_ -eq 'linguistics' -or $_ -eq 'physics' -or $_ -eq 'commerce' -or $_ -eq 'hobby' -or $_ -eq 'plant' -or $_ -eq 'weather' -or $_ -eq 'geographics' -or $_ -eq 'Eurasia' -or $_ -eq 'North America' -or $_ -eq 'New England' -or $_ -eq 'electrical device' -or $_ -eq 'building' -or $_ -eq 'instrument' -or $_ -eq 'draughts' -or $_ -eq 'game' -or $_ -eq 'theatre' -or $_ -eq 'film' -or $_ -eq 'body language' -or $_ -eq 'sound perception' -or $_ -eq 'seeing' -or $_ -eq 'mining' -or $_ -eq 'animal husbandry' -or $_ -eq 'physiological condition' -or $_ -eq 'Asia' -or $_ -eq 'South China Sea' -or $_ -eq 'Tory Burch' -or $_ -eq 'animal' -or $_ -eq 'recreation' -or $_ -eq 'elevation' -or $_ -eq 'Africa' -or $_ -eq 'nation' -or $_ -eq 'speech') { continue }
      
      if ($_ -eq 'fashion') {
        if ($topics -notlike '*cloth*') { 
          $topics = $topics + "/clothing"
        }
        continue 
      }
      
      if ($_ -eq 'medicine') {
        $topics = $topics + " health"
      } else {
        $topics = $topics + "/" + $_
      }
    }
    if ($topics.Length -gt 1) {
      $topics = $topics.Substring(1)
    }
    #$topics = [system.String]::Join(", ", $parsedTisane.topics)
  }
  $hashtags = ''
  $orgs = ''
  $brands = ''
    
  if ($parsedTisane.entities_summary) {
    $parsedTisane.entities_summary | Sort-Object -Property name | Foreach-Object {
      $entityName = $_.name
      if ($_.ref_lemma) {
        $entityName = $_.ref_lemma
      }
      $namePattern = '*' + $entityName + '*'
      #$namePattern
      if ($orgs -notlike $namePattern -and ($_.type -eq 'organization' -or $_.type[0] -eq 'organization' -or $_.type[1] -eq 'organization')) {
        $orgs = $orgs + "/" + $entityName
      }
      if ($hashtags -notlike $namePattern -and ($_.type -eq 'hashtag' -or $_.type[0] -eq 'hashtag' -or $_.type[1] -eq 'hashtag')) {
        $hashtags = $hashtags + " " + $entityName
      }
      if ($brands -notlike $namePattern -and ($_.type -eq 'product' -or $_.type[0] -eq 'product' -or $_.type[1] -eq 'product')) {
        $brands = $brands + "/" + $entityName
      }    
    }
    if ($orgs.Length -gt 1) {
      $orgs = $orgs.Substring(1)
    }
    if ($brands.Length -gt 1) {
      $brands = $brands.Substring(1)
    }
  }
  
  if ($parsedTisane.sentiment_expressions) {
    $parsedTisane.sentiment_expressions | Foreach-Object {
      if ($_.targets -and $_.targets[0] -ne 'large' -and $_.targets[0] -ne 'long' -and $_.targets[0] -ne 'ethics' -and $_.targets[0] -ne 'privacy' -and $_.targets[0] -ne 'packaging' -and $_.targets[0] -ne 'user_friendliness' -and $_.targets[0] -ne 'decoration') {
        $ptrn = '*' + $_.targets[0] + '*'
        if ($aspects -notlike $ptrn ) {
          $sortedTargets = $_.targets | Sort-Object
          $aspects = $aspects + "/" + [system.String]::Join("/", $sortedTargets)
        }
      }
      if ($_.reasons -and $_.reasons[0] -ne 'large' -and $_.reasons[0] -ne 'long' -and $_.reasons[0] -ne 'ethics' -and $_.reasons[0] -ne 'privacy' -and $_.reasons[0] -ne 'packaging' -and $_.reasons[0] -ne 'user_friendliness' -and $_.reasons[0] -ne 'decoration') {
       $ptrn = '*' + $_.reasons[0] + '*'
       if ($aspects -notlike $ptrn) {
         $sortedReasons = $_.reasons | Sort-Object
         $aspects = $aspects + "/" + [system.String]::Join("/", $sortedReasons)
        }
      }
    }
    if ($aspects.Length -gt 1) {
      $aspects = $aspects.Substring(1)
    }
  }

  $sentimentDescription = ''
  if ($content -like '*Top *' -or $content -like '*Campaign*') {
    #$srcSheet.Cells.Item($i,$promotionColumnIndex).Value = 'promotion'
    $sentimentDescription = 'positive (promotion)'
  } else {
    if ($parsedTisane.sentiment -lt 0) {
      $sentimentDescription = 'negative'
    } else {
      if ($parsedTisane.sentiment -gt 0) {
          $sentimentDescription = 'positive'
      }
    }
  }
  "Sentiment: $sentimentDescription Aspects: $aspects Brands: $brands Orgs: $orgs Hashtags: $hashtags Topics: $topics"
  $srcSheet.Cells.Item($i,$sentimentColumnIndex).Value = $sentimentDescription
  $srcSheet.Cells.Item($i,$abuseColumnIndex).Value = "$abuseType $abuseTags".Trim()
  $srcSheet.Cells.Item($i,$topicColumnIndex).Value = $topics.Trim()
  $srcSheet.Cells.Item($i,$aspectsColumnIndex).Value = $aspects.Trim()
  $srcSheet.Cells.Item($i,$brandColumnIndex).Value = $brands.Trim()
  $srcSheet.Cells.Item($i,$orgColumnIndex).Value = $orgs.Trim()
  $srcSheet.Cells.Item($i,$hashtagsColumnIndex).Value = $hashtags.Trim()
  #if ($abuseType) {
  # "$abuseType ($abuseText) $abuseText / : $content"
  #}
}

Write-Progress -Activity "Almost done" -Status "Saving the spreadsheet"


$Workbook.Save()
$workbook.Close($false)
[void][System.Runtime.InteropServices.Marshal]::ReleaseComObject([System.__ComObject]$Excel)
[gc]::Collect()
[gc]::WaitForPendingFinalizers()
Remove-Variable excel -ErrorAction SilentlyContinue
}
#EndRegion '.\Public\Excel-BrandTest.ps1' 216
#Region '.\Public\Excel-ModerationTest.ps1' 0
function Excel-ModerationTest{
  [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Language(s): ")][String] $language,
     [Parameter(Mandatory = $true, HelpMessage="Spreadsheet filename: ")][String] $filename,
     [Parameter(Mandatory = $true, HelpMessage="1st output column: ")][int] $outputColumnIndex,
     [Parameter(Mandatory = $true, HelpMessage="Start line: ")][int] $startLine,
     [Parameter(Mandatory = $false, HelpMessage="End line: ")][int] $endLine,
     [Parameter(Mandatory = $false, HelpMessage="Sheet index: ")][int] $sheetIndex,
     [Parameter(Mandatory = $false, HelpMessage="Input column: ")][int] $inputColumn,
     [Parameter(Mandatory = $false, HelpMessage="Negativity: ")][bool] $negativity
)


# $TISANE_PATH = 'C:/Tisane/TestConsole/'
$ROW_COUNT_IN_SPREADSHEET = 3 #1000

$OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding =
                    New-Object System.Text.UTF8Encoding

$Excel = New-Object -ComObject Excel.Application
$Workbook = $Excel.Workbooks.Open($filename)
if (-not($sheetIndex)) {
  $sheetIndex = 1
}
if (-not($outputColumnIndex)) {
  $outputColumnIndex = 3
}
if (-not($inputColumn)) {
  $inputColumn = 1
}
$srcSheet = $Workbook.Sheets.Item($sheetIndex)
# $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"

# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path) # assign the configuration file

# # BEGIN fix for Powershell bug: in some cases, the configuration files aren't read properly
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# # END fix for Powershell bug: in some cases, the configuration files aren't read properly

# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll") # load the type

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
Write-Progress -Activity "Loading language model" -Status "Normally takes 30-40 sec"
[Tisane.Server]::ActivateLazyLoading()
$doesntMatter = $tisane.Parse('en', 'Testing', '{}')
if (-not($startLine)) {
  $startLine = 2
}
if (-not($endLine)) {
  $endLine = $startLine + $ROW_COUNT_IN_SPREADSHEET
}

if ($negativity) {
  $settings = '{"snippets":true,"topic_standard":"native","sentiment":true,"document_sentiment":true,"parses":false,"words":true,"entities":true,"explain":true,"format":"dialogue1"}'
} else {
  $settings = '{"snippets":true,"topic_standard":"native","sentiment":false,"parses":false,"words":true,"entities":true,"explain":true,"format":"dialogue1"}'
}


For ($i=$startLine; $i -le $endLine; $i++) {
  $content = $srcSheet.Cells.Item($i,$inputColumn).Text
  if (-not $content) {continue}
  $languageJson = $tisane.DetectLanguage($content, $language, '')
  $languageDetection = $languageJson | ConvertFrom-Json
  
  
  $pct = (($i - $startLine) / ($endLine - $startLine)) * 100
  Write-Progress -Activity "[$i] $content" -Status "$pct% complete" -PercentComplete $pct
  $currentLanguage = $language
  if ($languageDetection.languages -and $languageDetection.languages[0].language -and $languageDetection.languages[0].language -ne 'mg' -and $languageDetection.languages[0].language -ne 'lt' -and $languageDetection.languages[0].language -ne 'un' -and $languageDetection.languages[0].language -ne 'cy' -and $languageDetection.languages[0].language -ne 'lt') {
    $currentLanguage = $languageDetection.languages[0].language
  }
  $tisaneResult = $tisane.Parse($currentLanguage, $content, $settings)
  #$tisaneResult
  $parsedTisane = ConvertFrom-Json -InputObject $tisaneResult
  #$parsedTisane | select -ExpandProperty sentence_list
  $abuseText = ''
  $abuseTags = ''
  $abuseType = ''
  $explanation = ''
  $tagColumn = ''
  if ($parsedTisane.abuse) {
    $parsedTisane.abuse | Foreach-Object {
      $abuseText = $_.text
      $abuseTags = $_.tags
      $abuseType = $_.type
      $explanation = $_.explanation
      $tagColumn = ''
      if (-not ($abuseType -eq 'criminal_activity' -or $abuseType -eq 'external_contact')) {
      if ($abuseTags) {
        $abuseTags | Foreach-Object {
          switch ($_) {
            'cryptocurrency' { $tagColumn = "crypto ₿ $tagColumn" }
            'scam' { $tagColumn = "fraud 🤥 $tagColumn" }
            'soft_drug' { $tagColumn = "drugs 🌿 $tagColumn" }
            'hard_drug' { $tagColumn = "drugs 💉 $tagColumn" }
            'medication' { $tagColumn = "drugs 💊 $tagColumn" }
            'death' { $tagColumn = "death 💀 $tagColumn" }
            'data' { $tagColumn = "data 💳 $tagColumn" }
            'violence' { $tagColumn = "violence 👊 $tagColumn" }
            'call_for_violence' { $tagColumn = "calls for violence 👊 $tagColumn" }
            'firearms' { $tagColumn = "firearms 🔫 $tagColumn" }
            'firearms' { $tagColumn = "explosives 💣 $tagColumn" }
            'sex' { $tagColumn = "sex 💋 $tagColumn" }
            'sex_work' { $tagColumn = "sex work 💋 $tagColumn" }
            'conspiracy_theory' { $tagColumn = "conspiracy theory 😵 $tagColumn" }
            default { $tagColumn = "$_ $tagColumn" }
          }
        } 
      }
      switch ($_.type) {
        'criminal_activity' {
          
          if (-not $tagColumn -and $parsedTisane.topics) {
            $parsedTisane.topics | Foreach-Object {
               switch ($_) {
                'narcotic' { $tagColumn = "drugs 💉" }
                'drug' { $tagColumn = "drugs 💉" }
                'soft drug' { $tagColumn = "drugs 🌿" }
                'hard drug' { $tagColumn = "drugs 💉" }
                'medication' { $tagColumn = "drugs 💊" }
                'threat' {$tagColumn = "threat 👿"}
                'planning' {$tagColumn = "planning ✍"}
                'sourcing' {$tagColumn = "procurement 📰"}
                'promotion' {$tagColumn = "promotion 📢"}
                'child abuse' {$tagColumn = "child abuse 🚸"}
                'animal' {$tagColumn = "wildlife and poaching 🦏"}
                'identity theft' {$tagColumn = "identity and data theft 💳"}
                'credit card' {$tagColumn = "identity and data theft 💳"}
                'firearm' {$tagColumn = "firearms 🔫"}
                'cryptocurrency' {$tagColumn = "cryptocurrency ₿"}
                'fraud' {$tagColumn = "fraud 🤥"}
                'explosive' {$tagColumn = "explosives 💣"}
                'explosive device' {$tagColumn = "explosives 💣"}
                  
              }
           }
          }
        }
        'data' {
          $tagColumn = $tagColumn + " identity and data theft 💳"
        }
# 'personal_attack' {
# }
# 'bigotry' {
# $hateSpeech = $hateSpeech + " " + $abuseText
# }
# 'sexual_advances' {
# $sexualAdvances = $sexualAdvances + " " + $abuseText
# }
# 'external_contacts' {
# $contacts = $contacts + " " + $abuseText
# }
        default {
          
        }
      }
    }
    }
    
  }
  
  if ($parsedTisane.entities_summary) {
    $parsedTisane.entities_summary | Foreach-Object {
      if ($_.type -eq 'software' -or $_.type[0] -eq 'software' -and $_.type[1] -ne 'website' -and $_.type[2] -ne 'website' -or $_.type[1] -eq 'software' -and $_.type[0] -ne 'website' -and $_.type[2] -ne 'website') {
        if ($software) {
          $software = $software + ' / ' + $_.name
        } else {
          $software = $_.name
        }
      }
      else {
        if ($_.type -eq 'place' -or $_.type[0] -eq 'place' -or $_.type[1] -eq 'place') {
          if ($locations) {
            $locations = $locations + ' / ' + $_.name
          } else {
            $locations = $_.name
          }
        }
        else {
          if ($_.type -eq 'organization' -or $_.type[0] -eq 'organization' -or $_.type[1] -eq 'organization') {
            if ($orgs) {
              $orgs = $orgs + ' / ' + $_.name
            } else {
              $orgs = $_.name
            }
          } else {
            if ($_.type -eq 'person' -or $_.type -eq 'username') {
              if ($people) {
                $people = $people + ' / ' + $_.name
              } else {
                $people = $_.name            
              }
            } else {
              if ($_.type -eq 'email' -or $_.type -eq 'username') {
                if ($contactDetails) {
                  $contactDetails = $contactDetails + ' / ' + $_.name
                } else {
                  $contactDetails = $_.name            
                }
              } else {
                $name = $_.name
                switch ($_.type) 
                { 
                  'time_range' { 
                    if ($time_ranges) {
                      $time_ranges = $time_ranges + ' / ' + $name
                    } else {
                      $time_ranges = $name
                    }
                  }
                  'date' { 
                    if ($dates) {
                      $dates = $dates + ' / ' + $name
                    } else {
                      $dates = $name
                    }
                  }
                  'time' { 
                    if ($times) {
                      $times = $times + ' / ' + $name
                    } else {
                      $times = $name
                    }
                  }
                  'file' {
                    if ($files) {
                      $files = $files + ' / ' + $name
                    } else {
                      $files = $name
                    }
                  }
                  'phone' {
                    if ($phones) {
                      $phones = $phones + ' / ' + $name
                    } else {
                      $phones = $name
                    }
                  }

                }
              }
            }
          }
        }
      }
    }
  }
  $srcSheet.Cells.Item($i,$outputColumnIndex).Value = $abuseType
  $srcSheet.Cells.Item($i,$outputColumnIndex + 1).Value = $tagColumn
  $srcSheet.Cells.Item($i,$outputColumnIndex + 2).Value = $abuseText
  # $srcSheet.Cells.Item($i,$outputColumnIndex + 3).Value = $explanation
  # $srcSheet.Cells.Item($i,$outputColumnIndex + 4).Value = $phones
  if ($negativity) {
    $documentSentiment = $parsedTisane.sentiment
    if ($documentSentiment -lt -0.5) {
      $srcSheet.Cells.Item($i,$outputColumnIndex + 3).Value = "negativity ☹"
    }
  }
  if ($abuseType) {
    "$abuseType ($abuseText) $abuseText / : $content"
  }
}

Write-Progress -Activity "Almost done" -Status "Saving the spreadsheet"


$Workbook.Save()
$workbook.Close($false)
[void][System.Runtime.InteropServices.Marshal]::ReleaseComObject([System.__ComObject]$Excel)
[gc]::Collect()
[gc]::WaitForPendingFinalizers()
Remove-Variable excel -ErrorAction SilentlyContinue
}
#EndRegion '.\Public\Excel-ModerationTest.ps1' 281
#Region '.\Public\Excel-TagAbuse.ps1' 0

# modify the 4 parameters below as you need. Assuming your samples are in the first column
function Excel-TagAbuse{
[CmdletBinding()]
# $SPREADSHEET_PATHNAME = 'c:/PATH_WHERE_I_STORE_MY_SPREADSHEETS/TisaneTest.xlsx'
# $TISANE_PATH = 'C:/Tisane/TestConsole/'
# $ROW_COUNT_IN_SPREADSHEET = 1000
# $languageCode = 'en' # assuming the spreadsheets are monolingual
Param(
      [Parameter(Mandatory = $true, HelpMessage="Spreadsheet path: Example: ('c:/PATH_WHERE_I_STORE_MY_SPREADSHEETS/TisaneTest.xlsx') ")][String] $SPREADSHEET_PATHNAME,
      # [Parameter(Mandatory = $true, HelpMessage="Tisane path: ()")][String] $TISANE_PATH,
      [Parameter(Mandatory = $false, HelpMessage="Row count in spreadsheet: (Default: 1000)")][int] $ROW_COUNT_IN_SPREADSHEET = 1000,
      [Parameter(Mandatory = $false, HelpMessage="Language code: (Default en) ")][String] $languageCode = 'en'
      )


$Excel = New-Object -ComObject Excel.Application
$Workbook = $Excel.Workbooks.Open($SPREADSHEET_PATHNAME)
$srcSheet = $Workbook.Sheets.Item(1)
# $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"

# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path) # assign the configuration file

# # BEGIN fix for Powershell bug: in some cases, the configuration files aren't read properly
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# # END fix for Powershell bug: in some cases, the configuration files aren't read properly

# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll") # load the type

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
Write-Progress -Activity "Loading language model" -Status "Normally takes 30-40 sec"
$doesntMatter = $tisane.Parse($languageCode, 'Testing', '{}')
$startLine = 2
$endLine = $ROW_COUNT_IN_SPREADSHEET
$outLine = 2


For ($i=$startLine; $i -le $endLine; $i++) {
  $content = $srcSheet.Cells.Item($i,1).Text
  if (-not $content) {continue}
  
  $pct = (($i - $startLine) / ($endLine - $startLine)) * 100
  Write-Progress -Activity "[$i] $content" -Status "$pct% complete" -PercentComplete $pct
  $tisaneResult = $tisane.Parse($languageCode, $content, '{"snippets":true,"topic_standard":"native","sentiment":false,"parses":false,"words":true,"entities":true}')
  $tisaneResult
  $parsedTisane = ConvertFrom-Json -InputObject $tisaneResult
  $crimeDomain = ''
  $criminalActivity = ''
  $personalAttacks = ''
  $hateSpeech = ''
  $contacts = ''
  $contactDetails = ''
  $sexualAdvances = ''
  $people = ''
  $locations = ''
  $time_ranges = ''
  $dates = ''
  $times = ''
  $files = ''
  $phones = ''
  $orgs = ''
  $software = ''
  if ($parsedTisane.abuse) {
    $parsedTisane.abuse | Foreach-Object {
      $abuseText = $_.text
      $abuseTags = $_.tags
      switch ($_.type) {
        'criminal_activity' {
          $crimePrefix = ''
          if ($abuseTags) {
            $abuseTags | Foreach-Object {
              if ($_ -ne 'addressee' -or $_ -ne 'quantitative') {
              if ($crimePrefix) {
                $crimePrefix = $crimePrefix + '/' + $_
              } else {
                $crimePrefix = $_
              }
              switch ($_) {
                'scam' { $crimeDomain = "fraud 🤥" }
                'soft_drug' { $crimeDomain = "drugs 🌿" }
                'hard_drug' { $crimeDomain = "drugs 💉" }
                'medication' { $crimeDomain = "drugs 💊" }
                'death' { $crimeDomain = "death 💀" }
                'data' { $crimeDomain = "identity and data theft 💳" }
              }
              }
            }
            
          }
          
          if (-not $crimeDomain -and $parsedTisane.topics) {
            $parsedTisane.topics | Foreach-Object {
               switch ($_) {
                'narcotic' { $crimeDomain = "drugs 💉" }
                'drug' { $crimeDomain = "drugs 💉" }
                'soft drug' { $crimeDomain = "drugs 🌿" }
                'hard drug' { $crimeDomain = "drugs 💉" }
                'medication' { $crimeDomain = "drugs 💊" }
                'threat' {$crimeDomain = "threat 👿"}
                'planning' {$crimeDomain = "planning ✍"}
                'sourcing' {$crimeDomain = "procurement 📰"}
                'promotion' {$crimeDomain = "promotion 📢"}
                'child abuse' {$crimeDomain = "child abuse 🚸"}
                'animal' {$crimeDomain = "wildlife and poaching 🦏"}
                'identity theft' {$crimeDomain = "identity and data theft 💳"}
                'credit card' {$crimeDomain = "identity and data theft 💳"}
                'firearm' {$crimeDomain = "firearms 🔫"}
                'cryptocurrency' {$crimeDomain = "cryptocurrency ₿"}
                'fraud' {$crimeDomain = "fraud 🤥"}
                'explosive' {$crimeDomain = "explosives 💣"}
                'explosive device' {$crimeDomain = "explosives 💣"}
                  
              }
           }
          }
          
          if ($crimePrefix -and $criminalActivity.IndexOf($crimePrefix) -lt 0) {
            $criminalActivity = $criminalActivity + ' [' + $crimePrefix + '] ' + $abuseText
          } else {
            $criminalActivity = $criminalActivity + ' ' + $abuseText
          }
        }
        'data' {
          $criminalActivity = $criminalActivity + ' ' + $abuseText
          $crimeDomain = $crimeDomain + " identity and data theft 💳"
        }
        'personal_attack' {
          $personalAttacks = $personalAttacks + " " + $abuseText
        }
        'bigotry' {
          $hateSpeech = $hateSpeech + " " + $abuseText
        }
        'sexual_advances' {
          $sexualAdvances = $sexualAdvances + " " + $abuseText
        }
        'external_contacts' {
          $contacts = $contacts + " " + $abuseText
        }
      }
    }
    
  }
  
  if ($parsedTisane.entities_summary) {
    $parsedTisane.entities_summary | Foreach-Object {
      if ($_.type -eq 'software' -or $_.type[0] -eq 'software' -and $_.type[1] -ne 'website' -and $_.type[2] -ne 'website' -or $_.type[1] -eq 'software' -and $_.type[0] -ne 'website' -and $_.type[2] -ne 'website') {
        if ($software) {
          $software = $software + ' / ' + $_.name
        } else {
          $software = $_.name
        }
      }
      else {
        if ($_.type -eq 'place' -or $_.type[0] -eq 'place' -or $_.type[1] -eq 'place') {
          if ($locations) {
            $locations = $locations + ' / ' + $_.name
          } else {
            $locations = $_.name
          }
        }
        else {
          if ($_.type -eq 'organization' -or $_.type[0] -eq 'organization' -or $_.type[1] -eq 'organization') {
            if ($orgs) {
              $orgs = $orgs + ' / ' + $_.name
            } else {
              $orgs = $_.name
            }
          } else {
            if ($_.type -eq 'person' -or $_.type -eq 'username') {
              if ($people) {
                $people = $people + ' / ' + $_.name
              } else {
                $people = $_.name            
              }
            } else {
              if ($_.type -eq 'email' -or $_.type -eq 'username') {
                if ($contactDetails) {
                  $contactDetails = $contactDetails + ' / ' + $_.name
                } else {
                  $contactDetails = $_.name            
                }
              } else {
                $name = $_.name
                switch ($_.type) 
                { 
                  'time_range' { 
                    if ($time_ranges) {
                      $time_ranges = $time_ranges + ' / ' + $name
                    } else {
                      $time_ranges = $name
                    }
                  }
                  'date' { 
                    if ($dates) {
                      $dates = $dates + ' / ' + $name
                    } else {
                      $dates = $name
                    }
                  }
                  'time' { 
                    if ($times) {
                      $times = $times + ' / ' + $name
                    } else {
                      $times = $name
                    }
                  }
                  'file' {
                    if ($files) {
                      $files = $files + ' / ' + $name
                    } else {
                      $files = $name
                    }
                  }
                  'phone' {
                    if ($phones) {
                      $phones = $phones + ' / ' + $name
                    } else {
                      $phones = $name
                    }
                  }

                }
              }
            }
          }
        }
      }
    }
  }
  $srcSheet.Cells.Item($i,2).Value = $criminalActivity
  $srcSheet.Cells.Item($i,3).Value = $crimeDomain
  $srcSheet.Cells.Item($i,4).Value = $personalAttacks
  $srcSheet.Cells.Item($i,5).Value = $hateSpeech
  $srcSheet.Cells.Item($i,6).Value = $sexualAdvances
  $srcSheet.Cells.Item($i,7).Value = $contacts
  $srcSheet.Cells.Item($i,8).Value = $people
  $srcSheet.Cells.Item($i,9).Value = $orgs
  $srcSheet.Cells.Item($i,10).Value = $software
  $srcSheet.Cells.Item($i,11).Value = $locations
  $srcSheet.Cells.Item($i,12).Value = $time_ranges
  $srcSheet.Cells.Item($i,13).Value = $dates
  $srcSheet.Cells.Item($i,14).Value = $times
  $srcSheet.Cells.Item($i,15).Value = $files
  $srcSheet.Cells.Item($i,16).Value = $phones
}

Write-Progress -Activity "Almost done" -Status "Saving the spreadsheet"


$Workbook.Save()
$workbook.Close($false)
[void][System.Runtime.InteropServices.Marshal]::ReleaseComObject([System.__ComObject]$Excel)
[gc]::Collect()
[gc]::WaitForPendingFinalizers()
Remove-Variable excel -ErrorAction SilentlyContinue
}
#EndRegion '.\Public\Excel-TagAbuse.ps1' 262
#Region '.\Public\List-UnknownLexemes.ps1' 0
function List-UnknownLexemes {
    [CmdletBinding()]
    Param(
        [Parameter(Mandatory = $true, HelpMessage="Language code: ")][String] $languageCode,
        [Parameter(Mandatory = $true, HelpMessage="Input filename: ")][String] $inputFilename,
        [Parameter(Mandatory = $true, HelpMessage="Minimum frequency: ")][int] $minimum
    )
    $tisane = Create-Tisane
    $i = 0

    # Process the CSV file line by line
    Import-Csv -Path $inputFilename | ForEach-Object {
        $entry = $_
        if ([int]$entry.'log-grade' -ge $minimum) {
            $word = $entry.word
            Write-Progress -Activity "Checking words" -Status "Processing: [$word]"
            if ($tisane.ListSenses($languageCode, $word).Length -le 2) {
                if (-not (($tisane.Parse($languageCode, $word, '{"words":true}') | ConvertFrom-Json).sentence_list.words.Length -ge 1)) {
                    Write-Output $word
                }
            }
            $i++
        }else{
            break
        }
    }
    Write-Progress -Activity "Checking words" -Status "Processing complete" -Completed
}
# List-UnknownLexemes -languageCode "en" -inputFilename "en_graded.csv" -minimum 7
#EndRegion '.\Public\List-UnknownLexemes.ps1' 30
#Region '.\Public\Parse-CustomJson.ps1' 0
function Parse-CustomJson{
  [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Language(s): ")][String] $language,
     [Parameter(Mandatory = $true, HelpMessage="Filename including path: ")][String] $path,
     [Parameter(Mandatory = $true, HelpMessage="Out filename including path: ")][String] $outFile,
     [Parameter(Mandatory = $false, HelpMessage="Corpora ID: ")][int] $skip
)


$fileSize = (Get-Item $path).Length
$PROGRESS_UPDATE_INTERVAL = 10
$lastReported = 0
$processed = 0
$settings = '{"snippets":true,"sentiment":false,"entities":false,"topics":false,"optimize_topics":false,"explain":true}'

# $TISANE_PATH = 'C:/Tisane/TestConsole/'
# $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"

[System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path) # assign the configuration file

# # BEGIN fix for Powershell bug: in some cases, the configuration files aren't read properly
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# # END fix for Powershell bug: in some cases, the configuration files aren't read properly

# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll") # load the type
$ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
[System.Diagnostics.Trace]::Listeners.Add($ttr)
[System.Diagnostics.Trace]::AutoFlush = $true

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane


Write-Progress -Activity "Building $outFile" -Status "$0% complete" -PercentComplete 0
#"Loading language models...`r`n"
# $tisane = New-Object Tisane.Server
$r = [IO.File]::OpenText($path)
$startedAt = Get-Date
"Everything is loaded. Starting at " + $startedAt.ToString('T')
$outContent = ''
$detected = 0
$control = 'not yet'
$totalPostCount = 0
while ($r.Peek() -ge 0) {
    $line = $r.ReadLine()
    #$processed += $line.Length
    # Process $line here...
    $inJson = ConvertFrom-Json -InputObject $line
    $objectCount = $inJson.length
    $inJson | ForEach-Object {
      $totalPostCount += 1
      $processed += $line.Length / $objectCount
        if (-not($skip) -or $totalPostCount -gt $skip) {
        if ($processed - $lastReported -gt $PROGRESS_UPDATE_INTERVAL) {
          #if ($processed % 500 -eq 0) {
            $outContent | Set-Content ($outFile) -encoding UTF8
          #}
          #$inJson.content
          $pct = ($processed / $fileSize) * 100
          $timeTaken = ((Get-Date) - $startedAt).TotalMinutes
          $abuseShare = $detected / $totalPostCount * 100
          Write-Progress -Activity "Building $outFile" -Status "$pct% complete, running for $timeTaken minutes, $detected instances ($abuseShare %) detected out of $totalPostCount posts" -PercentComplete $pct

          $lastReported = $processed
        }
        $response = $tisane.Parse($language, $_.body, $settings)
        $response = '{"uuid": "' + $_.uuid + '", ' + $response.Substring(1)
        $responseParsed = ConvertFrom-Json -InputObject $response
        if ($response -cmatch '"abuse"') {
          $meaningfulAbuseCount = 0
          $responseParsed.abuse | ForEach-Object {
            $abuseType = $_.type
            #"Abuse type: $abuseType"
            if ($abuseType -eq 'bigotry' -or $abuseType -eq 'personal_attack' -or $abuseType -eq 'profanity' -or ($abuseType -eq 'criminal_activity' -and $_.tags -and $_.tags[0] -eq 'call_to_violence')) {
              $meaningfulAbuseCount += 1
              #"Yes, $abuseType fits!"
            }
          }
          if ($meaningfulAbuseCount -gt 0) {
            $detected += 1
            "Abuse detected: " + $response + "`r`n"
          }
        }
        $outContent = $outContent + [Environment]::NewLine + $response
      }
    }
}
#"Control: $control"
$r.Dispose()
# save content to a new file
$outContent | Set-Content ($outFile) -encoding UTF8
$finishedAt = Get-Date
$timeTaken = ($finishedAt - $startedAt).TotalMinutes
"Done at $finishedAt after $timeTaken minutes. $detected instance(s) detected"
}
#EndRegion '.\Public\Parse-CustomJson.ps1' 101
#Region '.\Public\Parse-Lines.ps1' 0
function Parse-Lines{
    [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Path: ")][String] $path,
     [Parameter(Mandatory = $true, HelpMessage="Output Path: ")][String] $outPath,
     [Parameter(Mandatory = $true, HelpMessage="Language code: ")][String] $languageCode
     )
$fileSize = (Get-Item $path).Length
$outContent = ''
$processed = 0
$settings = '{"snippets":true,"sentiment":false,"entities":true,"topics":true,"optimize_topics":true,"topic_standard":"wikidata","explain":true}'
# $tisane_path = "C:/Tisane/TestConsole"
# $config_path = "$tisane_path/Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom("$tisane_path/Tisane.Runtime.dll")
Write-Progress -Activity "Building $outFile" -Status "$0% complete" -PercentComplete 0
#"Loading language models...`r`n"
$ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
[System.Diagnostics.Trace]::Listeners.Add($ttr)
[System.Diagnostics.Trace]::AutoFlush = $true

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
$r = [IO.File]::OpenText($path)
$startedAt = Get-Date
"Everything is loaded. Starting at " + $startedAt.ToString('T')
while ($r.Peek() -ge 0) {
    $line = $r.ReadLine()
    $processed += $line.Length
    # Process $line here...
    $tisaneResponse = $tisane.Parse($languageCode, $line, $settings)
    $outContent += "`r`n"
    $outContent += $tisaneResponse
    $pct = ($processed / $fileSize) * 100
    $timeTaken = ((Get-Date) - $startedAt).TotalMinutes
    Write-Progress -Activity "Building $outFile" -Status "$pct% complete, running for $timeTaken minutes" -PercentComplete $pct
}
#"Control: $control"
$r.Dispose()
$outContent
# save content to a new file
$outContent | Set-Content ($outPath) -encoding UTF8
$finishedAt = Get-Date
$timeTaken = ($finishedAt - $startedAt).TotalMinutes
"Done at $finishedAt after $timeTaken minutes. $detected instance(s) detected"
}
#EndRegion '.\Public\Parse-Lines.ps1' 52
#Region '.\Public\Reset-LocalTisaneSettings.ps1' 0
Function Reset-LocalTisaneSettings(){
    Save-LocalTisaneSettings -settingName 'TISANE_PATH' -settingValue $null
}
#EndRegion '.\Public\Reset-LocalTisaneSettings.ps1' 4
#Region '.\Public\Similarity-Test.ps1' 0
function Similarity-Test{
[CmdletBinding()]
# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  $languageCode = 'en'
  "Similarity1: " +  [Math]::Round($tisane.Similarity('en', 'there are 4 lights', 'en', 'there are 5 lights', '{}'),3)
  "Similarity2: " +  [Math]::Round($tisane.Similarity('en', 'there are four lights', 'en', 'there are 4 lights', '{}'),3)
  "Similarity3: " +  [Math]::Round($tisane.Similarity('en', 'dog eats a wolf', 'en', 'wolf eats a dog', '{}'),3)
  "Similarityx: " +  [Math]::Round($tisane.Similarity('en', 'a dog is eating a bone', 'en', 'a dog ate a bone', '{}'),3)
  #$comparisonResult = $tisane.CompareEntities('ru', 'Сергей Царнаев', 'ru', 'Серж Царнаев', 'person')
  #"Comparison 1:" + $comparisonResult
  #$comparisonResult = $tisane.CompareEntities('en', 'Mr Vadim Berman', 'en', 'Prof Vadim Berman', 'person')
  #"Comparison 2:" + $comparisonResult
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Similarity-Test.ps1' 31
#Region '.\Public\Tisane-CleanupTest.ps1' 0
function Tisane-CleanupTest{
  [CmdletBinding()]
Param(
     [Parameter(HelpMessage="Input file")][String] $pathname
)

# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# Add-Type -AssemblyName System.Web
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")

$content = Get-Content $pathname -encoding UTF8 | Out-String
#$content = [System.Web.HttpUtility]::HtmlDecode($content).Replace('"', '"')
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  $tisane.Normalize($tisane.ExtractText($content))
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-CleanupTest.ps1' 30
#Region '.\Public\Tisane-DebugTest.ps1' 0
function Tisane-DebugTest{
  [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Language(s): ")][String] $language,
     [Parameter(Mandatory = $true, HelpMessage="Content: ")][String] $content,
     [Parameter(Mandatory = $false, HelpMessage="Phrase to trace: ")][int] $traceId,
     [Parameter(Mandatory = $false, HelpMessage="Segment delimiter: ")][String] $delimiter,
     [Parameter(Mandatory = $false, HelpMessage="Memory flags: ")][String] $memoryFlags,
     [Parameter(Mandatory = $false, HelpMessage="Replace family: ")][int] $replaceFamily,
     [Parameter(Mandatory = $false, HelpMessage="Replace family with: ")][int] $replaceFamilyWith
)
# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
#[Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.LaMP.Data.dll")

#$ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
#[System.Diagnostics.Trace]::Listeners.Add($ttr)
$cnsltr = New-Object System.Diagnostics.ConsoleTraceListener
[System.Diagnostics.Trace]::Listeners.Add($cnsltr)
[System.Diagnostics.Trace]::AutoFlush = $true
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  [Tisane.Server]::ActivateLazyLoading()
  #$tisane.Parse('ru', '1')
  #$tisane.Parse('ko', '1')
  $memoryReplace = ''
  if ($replaceFamily -and $replaceFamilyWith) {
    $memoryReplace = ' "assign": [{"if":{"family":' + $replaceFamily + '}, "then":{"family":' + $replaceFamilyWith + '}}]'
  }
  $memory = ''
  if ($memoryFlags) {
    $featureList = [Tisane.Server]::FeatureListToJson($memoryFlags)
    if ($memoryReplace) {
      $memoryReplace = ", $memoryReplace"
    }
    $memory = ',"memory":{"flags":' + $featureList + $memoryReplace + '}'
    #"Memory: $memory"
  } else {
    $memory = ',"memory":{' + $memoryReplace + '}'
  }
  if ($traceId -gt 0) {
    $settings = '{"min_generic_frequency":1,"trace_id":' + $traceId + ',"trace_type":"phrase","debug":true,"confidence":true,"words":true,"parses":true,"snippets":true,"domain_match_threshold":1,"topic_standard":"iab_description","lang_detect_segmentation_regex":"' + $delimiter + '"' + $memory + '}'
  } else {
    $settings = '{"min_generic_frequency":1,"snippets":true,"confidence":true,"words":true,"domain_match_threshold":1,"topic_standard":"iab_description","lang_detect_segmentation_regex":"' + $delimiter + '","subscope":true' + $memory + '}'
  }
  #$res = $tisane.Parse('en|ko', 'u r a moron ! ты просто мудак', $settings)
  $res = $tisane.Parse($language, $content, $settings)
  $res | ConvertFrom-Json # | Select-Object sentence_list,parse_tree,words #-Property *
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-DebugTest.ps1' 64
#Region '.\Public\Tisane-DetectTest.ps1' 0
function Tisane-DetectTest{
  [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Content: ")][String] $content,
     [Parameter(Mandatory = $false, HelpMessage="Likely languages: ")][String] $languages,
     [Parameter(Mandatory = $false, HelpMessage="Segment delimiter: ")][String] $delimiter
     )

# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
#$ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
#[System.Diagnostics.Trace]::Listeners.Add($ttr)
[System.Diagnostics.Trace]::AutoFlush = $true
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  $res = $tisane.DetectLanguage($content, $languages, $delimiter)
  $res
  $res | ConvertFrom-Json | select -ExpandProperty languages | Format-Table -AutoSize -Property language,@{n='Fragment';e={$content.Substring($_.offset,$_.length - 1)}},offset,length
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-DetectTest.ps1' 34
#Region '.\Public\Tisane-FileTest.ps1' 0
function Tisane-FileTest{
  [CmdletBinding()]
Param(
     [Parameter(HelpMessage="Input file")][String] $pathname,
     [Parameter(HelpMessage="Language")][String] $language
)

# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# Add-Type -AssemblyName System.Web
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")

$content = Get-Content $pathname -encoding UTF8 | Out-String
#$content = [System.Web.HttpUtility]::HtmlDecode($content).Replace('"', '"')
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  #[Tisane.Server]::ActivateLazyLoading()
  $startedAt = Get-Date
  $input = $content
  $plainText = $tisane.ExtractText($content)
  $cleanedUp = $tisane.Normalize($plainText)
  #$json = $tisane.DetectLanguage($content, $language, '')
  #$json
  $json = $tisane.Parse($language, $cleanedUp, '{"parses":true,"words":true,"entities":false,"topics":false,"disable_spellcheck":true,"fast_sense_select":true,"disable_commonsense_cues":true}')
  $finishedAt = Get-Date
  $timeTaken = ($finishedAt - $startedAt).TotalSeconds
  $cleanedUpLength = $cleanedUp.Length
  $contentLength = $input.Length
  $bpsCleaned = $cleanedUpLength / $timeTaken
  $bps = $contentLength / $timeTaken
  #$json
  "Done in $timeTaken seconds. Speed for raw content ($contentLength): $bps chars per second, cleaned up ($cleanedUpLength): $bpsCleaned chars per second"
  "Cleaned up: $cleanedUp"
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-FileTest.ps1' 47
#Region '.\Public\Tisane-FilterEntityMismatches.ps1' 0
function Tisane-FilterEntityMismatches {
Param(
     [Parameter(HelpMessage="Language")][String] $languageCode,
     [Parameter(HelpMessage="Entity type")][String] $entityType,
     [Parameter(HelpMessage="Input file")][String] $inputFile,
     [Parameter(HelpMessage="Output file")][String] $outputFile,
     [Parameter(HelpMessage="Maximum count of mismatches")][int] $maxMismatchCount
     
)
# modify the 4 parameters below as you need. Assuming your samples are in the first column

# $TISANE_PATH = 'C:/Tisane/TestConsole/'
$DEFAULT_MAX_MISMATCH_COUNT = 5000
if (-not $maxMismatchCount -or $maxMismatchCount -lt 1) {
  $maxMismatchCount = $DEFAULT_MAX_MISMATCH_COUNT
}

# $config_path = $TISANE_PATH + "Tisane.TestConsole.exe.Config"

# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path) # assign the configuration file

# # BEGIN fix for Powershell bug: in some cases, the configuration files aren't read properly
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# # END fix for Powershell bug: in some cases, the configuration files aren't read properly

# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll") # load the type

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
Write-Progress -Activity "Loading language model" -Status "Normally takes 30-40 sec"

$undetectedCount = 0

$fileLines = Get-Content $inputFile

$outContent = ''
$fileLines | ForEach-Object {
  $tisaneResult = $tisane.Parse($languageCode, $_, '{"snippets":true,"topic_standard":"native","sentiment":false,"parses":false,"words":true,"entities":true,"format":"dialogue","explain":true}')
  $parsedTisane = ConvertFrom-Json -InputObject $tisaneResult
   
  $pct = $i / $fileLines.length * 100
  $i += 1
  $detectionRatio = ($i - $undetectedCount) / $i * 100
  Write-Progress -Activity "Processing" -Status "$pct% $_ not detected: $undetectedCount ($detectionRatio%)" -PercentComplete $pct
  
  if (-not $parsedTisane.entities_summary -or $parsedTisane.entities_summary[0].type -ne $entityType -or $parsedTisane.entities_summary[0].name.Length -lt $_.Length) {
    #$length1 = $parsedTisane.entities_summary[0].name.Length
    #$length2 = $_.Length
    #$currentType = $parsedTisane.entities_summary[0].type
    Write-Host "$_ not detected properly!" -ForegroundColor Red
    $undetectedCount += 1
    #$parsedTisane.entities_summary[0]
    $outContent = $outContent + [Environment]::NewLine + $_
    if ($undetectedCount -gt $maxMismatchCount) {
      $outContent | Set-Content $outputFile -encoding UTF8
      Write-Host "$maxMismatchCount reached, stopping. Detection rate: $detectionRatio%" -ForegroundColor Yellow
      Break
    }           
  } else {
    Write-Host "$_ detected properly!" -ForegroundColor Green
  }
}

$outContent | Set-Content $outputFile -encoding UTF8
}
#EndRegion '.\Public\Tisane-FilterEntityMismatches.ps1' 70
#Region '.\Public\Tisane-ParseDebugLog.ps1' 0
# Function to process the current entry
function Tisane-ParseDebugLog {
    Param(
        [Parameter(Mandatory = $true, HelpMessage="Input file path name: ")][String] $inputPathname,
        [Parameter(Mandatory = $true, HelpMessage="Output file path name: ")][String] $outputPathname
    )
    # $csvObjects = @()
    # $tisane_path = "C:/Tisane/TestConsole"
    # $config_path = "$tisane_path/Tisane.TestConsole.exe.config"
    # [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
    # Add-Type -AssemblyName System.Configuration
    # [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
    # [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
    # ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
    # [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
    # [Reflection.Assembly]::LoadFrom("$tisane_path/Tisane.Runtime.dll")
    # Write-Host "Everything is loaded: "
    # $tisane = New-Object Tisane.Server
    $tisane = Create-Tisane
    $logContent = Get-Content -Path $inputPathname -Encoding UTF8
    $fileSize = $logContent.Length
    $concatenatedLines = $logContent[0]
    $regexParsing = "Parsing\s+'(\w+)'"
    $regexContent = "content:\s+'([^']*)'"
    $regexSettings = "\{.*\}$"
    for ($i = 1; $i -lt $logContent.Length; $i++) {
        $currentLine = $logContent[$i]
        if ($currentLine.StartsWith("Tisane.Runtime.Service.exe")) {
            # Write-Host $concatenatedLines
            $matchParsing = [regex]::Match($concatenatedLines, $regexParsing)
            $matchContent = [regex]::Match($concatenatedLines, $regexContent)
            $settings = [regex]::Match($concatenatedLines, $regexSettings)
            $languageCode = $matchParsing.Groups[1].Value
            $text = $matchContent.Groups[1].Value
            if (-not([string]::IsNullOrEmpty($text))){
                Write-Host "Running tisane.Parse for: " -ForegroundColor Green
                Write-Host "language code : $languageCode"
                Write-Host "content: $text"
                Write-Host "settings: $settings"
                $tisaneResponse = $tisane.Parse($languageCode, $text, $settings)
                $line = "$tisaneResponse `n"
                Add-Content -Path $outputPathname -Value $line -Encoding UTF8
                $pct = ($i / $fileSize)*100
                # Write-Host $pct
                Write-Progress -Activity "Processing content: $text" -Status "$pct% " -PercentComplete $pct
            }
            # Add-Content -Path $outputPathname -Value $concatenatedLines #testing
            $concatenatedLines = $currentLine
        } else {
            $concatenatedLines += $currentLine
        }
    }

}

# Tisane-ParseDebugLog -inputPathname "tisaneExample.log" -outputPathname "o.log"
#EndRegion '.\Public\Tisane-ParseDebugLog.ps1' 57
#Region '.\Public\Tisane-PictureTest.ps1' 0
function Tisane-PictureTest{
  [CmdletBinding()]
# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
# $ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
# [System.Diagnostics.Trace]::Listeners.Add($ttr)
# [System.Diagnostics.Trace]::AutoFlush = $true
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  $r = $tisane.Parse('en', 'hollo world & pzza and tomotoes', '{"min_generic_frequency":2,"words":true}')
  $r
  $imageUrl = $tisane.Picture('ru', 'неизвестно, кто убийца')
  "Image 1: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'Copenhagen is not in Sweden, you know')
  "Image 2: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'I shot the sherrif but I never touched the deputy')
  "Image 3: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'tortellini with beef')
  "Image 4: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'pizza margerita')
  "Image 5: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'mars is far away from alpha centauri')
  "Image 6: $imageUrl"
  $imageUrl = $tisane.Picture('en', 'mars is a roman god of war')
  "Image 7: $imageUrl"
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-PictureTest.ps1' 41
#Region '.\Public\Tisane-SingleFileTest.ps1' 0
function Tisane-SingleFileTest{
[CmdletBinding()]
Param(
     [Parameter(HelpMessage="Input file")][String] $pathname,
     [Parameter(HelpMessage="Language")][String] $language
)

# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# Add-Type -AssemblyName System.Web
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")

$content = Get-Content $pathname -encoding UTF8 | Out-String
#$content = [System.Web.HttpUtility]::HtmlDecode($content).Replace('"', '"')
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  #[Tisane.Server]::ActivateLazyLoading()
  $input = $content
  $plainText = $tisane.ExtractText($content)
  $cleanedUp = $tisane.Normalize($plainText)
  $json = $tisane.Parse($language, $cleanedUp, '{"parses":true,"words":true,"entities":false,"topics":false,"disable_spellcheck":true,"fast_sense_select":true,"disable_commonsense_cues":true}')
  $json
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-SingleFileTest.ps1' 36
#Region '.\Public\Tisane-TranslateTest.ps1' 0
function Tisane-TranslateTest{
  [CmdletBinding()]
Param(
     [Parameter(Mandatory = $true, HelpMessage="Source language(s): ")][String] $srcLang,
     [Parameter(Mandatory = $true, HelpMessage="Target language(s): ")][String] $tarLang,
     [Parameter(Mandatory = $true, HelpMessage="Content: ")][String] $content,
     [Parameter(Mandatory = $false, HelpMessage="Entity type: ")][String] $entity,
     [Parameter(Mandatory = $false, HelpMessage="Assignment: ")][String] $assign
)
# $tisane_path = "C:/Tisane/TestConsole/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
#$ttr = New-Object System.Diagnostics.TextWriterTraceListener("C:/Tisane/ps1test.log")
#[System.Diagnostics.Trace]::Listeners.Add($ttr)
[System.Diagnostics.Trace]::AutoFlush = $true
$startedAt = Get-Date
try {
  # $tisane = New-Object Tisane.Server
  $tisane = Create-Tisane
  [Tisane.Server]::ActivateLazyLoading()
  #$res = $tisane.Transform('ko', 'en|ru|he|de', '운동은 하는 경우', '{}')
  if ($entity) {
    $settings = '{"entity":"' + $entity + '"' + $assign + '}'
  } else {
    $settings = "{$assign}"
  }
  $res = $tisane.Transform($srcLang, $tarLang, $content, $settings)
  $res
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-TranslateTest.ps1' 41
#Region '.\Public\Tisane-UnknownTest.ps1' 0
function Tisane-UnknownTest {
  Param(
     [Parameter(Mandatory = $false, HelpMessage="Language code (Default en): ")][String] $languageCode = 'en',
     [Parameter(Mandatory = $false, HelpMessage="Text to parse (Default : People like you should be executed gbrmv): ")][String] $textToParse = 'People like you should be executed gbrmv'
)
  
# $tisane_path = "C:/Tisane/"
# $config_path = $tisane_path + "Tisane.TestConsole.exe.config"
# [System.AppDomain]::CurrentDomain.SetData("APP_CONFIG_FILE", $config_path)
# Add-Type -AssemblyName System.Configuration
# [Configuration.ConfigurationManager].GetField("s_initState", "NonPublic, Static").SetValue($null, 0)
# [Configuration.ConfigurationManager].GetField("s_configSystem", "NonPublic, Static").SetValue($null, $null)
# ([Configuration.ConfigurationManager].Assembly.GetTypes() | where {$_.FullName -eq "System.Configuration.ClientConfigPaths"}).GetField("s_current", "NonPublic, Static").SetValue($null, $null)
# [Configuration.ConfigurationManager]::ConnectionStrings[0].Name
# [Reflection.Assembly]::LoadFrom($tisane_path + "Tisane.Runtime.dll")
$startedAt = Get-Date
try {

# $tisane = New-Object Tisane.Server
$tisane = Create-Tisane
$languageCode = 'en'
$textToParse = 'People like you should be executed gbrmv'
$tisaneResult = $tisane.Parse($languageCode, $textToParse, '{"snippets":true,"topic_standard":"native","sentiment":false,"parses":false,"words":true, "debug":true,"entities":true,"disable_spellcheck":true}')
$tisaneResult
$finishedAt = Get-Date
$timeTaken = ($finishedAt - $startedAt).TotalSeconds
$parsedTisane = ConvertFrom-Json -InputObject $tisaneResult
$parsedTisane.sentence_list | Foreach-Object {
    "Sentence: " + $_.text
    $_.words | Foreach-Object {
    "Word: " + $_.text + " " + $_.corrected_text
    if ($_.corrected_text -or $_.lexeme -eq 0) {
      'Corrected: ' + $_
    }
  }
}
"Done $timeTaken seconds"
}
catch {
  # $_ | Select-Object -Property *
  Write-Host $_.Exception
}
}
#EndRegion '.\Public\Tisane-UnknownTest.ps1' 44