Autotranslation using Locale::TextDomain::OO

About

• Translate messages of an external single language system
• Connect a translation API
• Handle abbreviations in text
• Correct translations
• Cache
• Limits or unavalibility of translation API

Bind dynamic lexicon

po file header

msgid ""
msgstr ""
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"X-Lexicon-Class: MyProject::I18N::AutoLexiconCached"

Abbreviations

po file content

#: MyProject::...::Class
msgid "EGB vernichtet"
msgstr "experience gift box wiped out"

Initialize

... = Locale::Utils::Autotranslator::ApiMymemoryTranslatedNet
  ->new(
    developer_language => $self->input_language,
    language           => $self->output_language,
    bytes_max          => $BYTES_MAX,
  );

Define called method

method fetch_from_lexicon ($lexicon_key, $message_key) {
  # $lexicon_key_ref e.g. { "category" => "de_cached",
  #                         "domain"   => "",
  #                         "language" => "de" }
  # $message_key_ref e.g. { "msgid" => "text" }
  my $key_util = Locale::TextDomain::OO::Util::JoinSplitLexiconKeys
    ->instance;
  my $lexicon_key_ref
    = $key_util->split_lexicon_key($lexicon_key);
  my $message_key_ref
    = $key_util->split_message_key($message_key);

Set language pair

  # languages
  my ($input_language)
    = $lexicon_key_ref->{category}
    =~ qr{ \A ( [a-z] [a-z-_]+ ) _cached \z }xms;
  $self->input_language(
    $input_language
    || confess 'Unexpected category ',
               $lexicon_key_ref->{category}
  );
  $self->output_language(
    $lexicon_key_ref->{language},
  );

Fallbacks

  # fallback
  my $msgid_return = { msgstr => $message_key_ref->{msgid} };

  # do not translate equal language pairs
  # e.g. de, de-at and de-ch to de
  $lexicon_key_ref->{language} =~ m{
    \A \Q$input_language\E \b
  }xms and return $msgid_return;

Length limit

  # too long for that API
  my $msgid_length = length decode_utf8(
    $message_key_ref->{msgid},
  );
  if ( $msgid_length > $BYTES_MAX ) {
    return $msgid_return;
  }

Call limit

  # define how often call that API
  my $last_updated = $self
    ->$resultset
    ->datetime_parser
    ->format_datetime(
      DateTime->now->subtract(
        seconds => $WAIT_SECONDS,
      ),
    );

Use cache database

  # found
  {
    my $found = $self
      ->$resultset
      ->search(
        {
          lexicon_key => $lexicon_key,
          message_key => $message_key,
        },
        {
          columns => [ qw(
            message_value reference
          ) ],
        },
      )
      ->one_row;

Found, Update cache

    if ( $found ) {
      $found->update({(
        lexicon_key => $lexicon_key,
        message_key => $message_key,
        updated     => $last_updated,
        $reference_pair->( $found->reference ),
      )});
      return {
        msgstr => $found->message_value
                  // $message_key_ref->{msgid},
      };
    }
  }

API blocked

  # not allowed to call the API
  {
    my $last_translation = $self
      ->$resultset
      ->search(
        {
          updated => {
            '>' => $last_updated,
          },
        },
        {
          columns  => 'updated',
          order_by => { -desc => 'updated' },
        },
      )
      ->one_row;

Blocked, register todo

    # but register translation
    if ( $last_translation ) {
      my $result = $self
        ->$resultset
        ->update_or_create({
          lexicon_key => $lexicon_key,
          message_key => $message_key,
        });
      $result->update({(
        lexicon_key => $lexicon_key,
        message_key => $message_key,
        updated     => $last_updated,
        $reference_pair->( $result->reference ),
      )});
      return $msgid_return;
    }
  }

Translate

  my $auto_translator = $self->auto_translator;
  my $translation
    = try {
      $auto_translator->translate_text(
        $message_key_ref->{msgid},
      );
    }

Translation failed, register todo

    catch {
      my $result = $self
        ->$resultset
        ->update_or_create({
          lexicon_key => $lexicon_key,
          message_key => $message_key,
          # updated   => defaults to now
        });

Translation failed, store reference

      my @reference_pair = $reference_pair->(
        $result->reference,
      );
      @reference_pair
        and $result->update({(
          lexicon_key => $lexicon_key,
          message_key => $message_key,
          @reference_pair,
        )});
      undef;
    };

Check translation

if ( length $translation ) {
  $translation =~ tr{\0\4}{};
  my $result = $self
    ->$resultset
    ->update_or_create({
      lexicon_key   => $lexicon_key,
      message_key   => $message_key,
      message_value => $translation,
      # updated     => defaults to now
    });

Store reference

    my @reference_pair = $reference_pair->(
      $result->reference,
    );
    @reference_pair
      and $result->update({(
        lexicon_key => $lexicon_key,
        message_key => $message_key,
        @reference_pair,
    )});
    return { msgstr => $translation };
  }

  return $msgid_return;
}

Hints

• Split large text into paragraphs
• Split large paragraphs into sentences
• Translation time can be long
• Use cronjob to solve registered but not translated

End

Thank you