/**************************************************************************** ** ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). ** All rights reserved. ** Contact: Nokia Corporation (qt-info@nokia.com) ** ** This file is part of the Qt Linguist of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:LGPL$ ** No Commercial Usage ** This file contains pre-release code and may not be distributed. ** You may use this file in accordance with the terms and conditions ** contained in the Technology Preview License Agreement accompanying ** this package. ** ** GNU Lesser General Public License Usage ** Alternatively, this file may be used under the terms of the GNU Lesser ** General Public License version 2.1 as published by the Free Software ** Foundation and appearing in the file LICENSE.LGPL included in the ** packaging of this file. Please review the following information to ** ensure the GNU Lesser General Public License version 2.1 requirements ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. ** ** In addition, as a special exception, Nokia gives you certain additional ** rights. These rights are described in the Nokia Qt LGPL Exception ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. ** ** If you have questions regarding the use of this file, please contact ** Nokia at qt-info@nokia.com. ** ** ** ** ** ** ** ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include "simtexth.h" #include "translator.h" #include #include #include QT_BEGIN_NAMESPACE typedef QList TML; /* How similar are two texts? The approach used here relies on co-occurrence matrices and is very efficient. Let's see with an example: how similar are "here" and "hither"? The co-occurrence matrix M for "here" is M[h,e] = 1, M[e,r] = 1, M[r,e] = 1, and 0 elsewhere; the matrix N for "hither" is N[h,i] = 1, N[i,t] = 1, ..., N[h,e] = 1, N[e,r] = 1, and 0 elsewhere. The union U of both matrices is the matrix U[i,j] = max { M[i,j], N[i,j] }, and the intersection V is V[i,j] = min { M[i,j], N[i,j] }. The score for a pair of texts is score = (sum of V[i,j] over all i, j) / (sum of U[i,j] over all i, j), a formula suggested by Arnt Gulbrandsen. Here we have score = 2 / 6, or one third. The implementation differs from this in a few details. Most importantly, repetitions are ignored; for input "xxx", M[x,x] equals 1, not 2. */ /* Every character is assigned to one of 20 buckets so that the co-occurrence matrix requires only 20 * 20 = 400 bits, not 256 * 256 = 65536 bits or even more if we want the whole Unicode. Which character falls in which bucket is arbitrary. The second half of the table is a replica of the first half, because of laziness. */ static const int indexOf[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ! " # $ % & ' ( ) * + , - . / 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0, // 0 1 2 3 4 5 6 7 8 9 : ; < = > ? 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15, // @ A B C D E F G H I J K L M N O 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, // P Q R S T U V W X Y Z [ \ ] ^ _ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, // ` a b c d e f g h i j k l m n o 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, // p q r s t u v w x y z { | } ~ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0, 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0 }; /* The entry bitCount[i] (for i between 0 and 255) is the number of bits used to represent i in binary. */ static const int bitCount[256] = { 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 }; struct CoMatrix { /* The matrix has 20 * 20 = 400 entries. This requires 50 bytes, or 13 words. Some operations are performed on words for more efficiency. */ union { quint8 b[52]; quint32 w[13]; }; CoMatrix() { memset( b, 0, 52 ); } CoMatrix(const QString &str) { QByteArray ba = str.toUtf8(); const char *text = ba.constData(); char c = '\0', d; memset( b, 0, 52 ); /* The Knuth books are not in the office only for show; they help make loops 30% faster and 20% as readable. */ while ( (d = *text) != '\0' ) { setCoOccurence( c, d ); if ( (c = *++text) != '\0' ) { setCoOccurence( d, c ); text++; } } } void setCoOccurence( char c, char d ) { int k = indexOf[(uchar) c] + 20 * indexOf[(uchar) d]; b[k >> 3] |= (1 << (k & 0x7)); } int worth() const { int w = 0; for ( int i = 0; i < 50; i++ ) w += bitCount[b[i]]; return w; } }; static inline CoMatrix reunion(const CoMatrix &m, const CoMatrix &n) { CoMatrix p; for (int i = 0; i < 13; ++i) p.w[i] = m.w[i] | n.w[i]; return p; } static inline CoMatrix intersection(const CoMatrix &m, const CoMatrix &n) { CoMatrix p; for (int i = 0; i < 13; ++i) p.w[i] = m.w[i] & n.w[i]; return p; } StringSimilarityMatcher::StringSimilarityMatcher(const QString &stringToMatch) { m_cm = new CoMatrix(stringToMatch); m_length = stringToMatch.length(); } int StringSimilarityMatcher::getSimilarityScore(const QString &strCandidate) { CoMatrix cmTarget(strCandidate); int delta = qAbs(m_length - strCandidate.size()); int score = ( (intersection(*m_cm, cmTarget).worth() + 1) << 10 ) / ( reunion(*m_cm, cmTarget).worth() + (delta << 1) + 1 ); return score; } StringSimilarityMatcher::~StringSimilarityMatcher() { delete m_cm; } /** * Checks how similar two strings are. * The return value is the score, and a higher score is more similar * than one with a low score. * Linguist considers a score over 190 to be a good match. * \sa StringSimilarityMatcher */ int getSimilarityScore(const QString &str1, const QString &str2) { CoMatrix cmTarget(str2); CoMatrix cm(str1); int delta = qAbs(str1.size() - str2.size()); int score = ( (intersection(cm, cmTarget).worth() + 1) << 10 ) / ( reunion(cm, cmTarget).worth() + (delta << 1) + 1 ); return score; } CandidateList similarTextHeuristicCandidates(const Translator *tor, const QString &text, int maxCandidates) { QList scores; CandidateList candidates; TML all = tor->translatedMessages(); foreach (const TranslatorMessage &mtm, all) { if (mtm.type() == TranslatorMessage::Unfinished || mtm.translation().isEmpty()) continue; QString s = mtm.sourceText(); int score = getSimilarityScore(s, text); if (candidates.size() == maxCandidates && score > scores[maxCandidates - 1] ) candidates.removeLast(); if (candidates.size() < maxCandidates && score >= textSimilarityThreshold) { Candidate cand( s, mtm.translation() ); int i; for (i = 0; i < candidates.size(); i++) { if (score >= scores.at(i)) { if (score == scores.at(i)) { if (candidates.at(i) == cand) goto continue_outer_loop; } else { break; } } } scores.insert(i, score); candidates.insert(i, cand); } continue_outer_loop: ; } return candidates; } QT_END_NAMESPACE