• Skip to content
  • Skip to link menu
  • KDE API Reference
  • kdelibs-4.14.38 API Reference
  • KDE Home
  • Contact Us
 

KDECore

  • kdecore
  • localization
  • probers
nsHebrewProber.cpp
Go to the documentation of this file.
1/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2/* -*- C++ -*-
3* Copyright (C) 1998 <developer@mozilla.org>
4*
5*
6* Permission is hereby granted, free of charge, to any person obtaining
7* a copy of this software and associated documentation files (the
8* "Software"), to deal in the Software without restriction, including
9* without limitation the rights to use, copy, modify, merge, publish,
10* distribute, sublicense, and/or sell copies of the Software, and to
11* permit persons to whom the Software is furnished to do so, subject to
12* the following conditions:
13*
14* The above copyright notice and this permission notice shall be included
15* in all copies or substantial portions of the Software.
16*
17* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24*/
25
26#include "nsHebrewProber.h"
27#include <stdio.h>
28
29// windows-1255 / ISO-8859-8 code points of interest
30#define FINAL_KAF ('\xea')
31#define NORMAL_KAF ('\xeb')
32#define FINAL_MEM ('\xed')
33#define NORMAL_MEM ('\xee')
34#define FINAL_NUN ('\xef')
35#define NORMAL_NUN ('\xf0')
36#define FINAL_PE ('\xf3')
37#define NORMAL_PE ('\xf4')
38#define FINAL_TSADI ('\xf5')
39#define NORMAL_TSADI ('\xf6')
40
41// Minimum Visual vs Logical final letter score difference.
42// If the difference is below this, don't rely solely on the final letter score distance.
43#define MIN_FINAL_CHAR_DISTANCE (5)
44
45// Minimum Visual vs Logical model score difference.
46// If the difference is below this, don't rely at all on the model score distance.
47#define MIN_MODEL_DISTANCE (0.01)
48
49#define VISUAL_HEBREW_NAME ("ISO-8859-8")
50#define LOGICAL_HEBREW_NAME ("windows-1255")
51
52namespace kencodingprober {
53bool nsHebrewProber::isFinal(char c)
54{
55 return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI));
56}
57
58bool nsHebrewProber::isNonFinal(char c)
59{
60 return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE));
61 // The normal Tsadi is not a good Non-Final letter due to words like
62 // 'lechotet' (to chat) containing an apostrophe after the tsadi. This
63 // apostrophe is converted to a space in FilterWithoutEnglishLetters causing
64 // the Non-Final tsadi to appear at an end of a word even though this is not
65 // the case in the original text.
66 // The letters Pe and Kaf rarely display a related behavior of not being a
67 // good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for
68 // example legally end with a Non-Final Pe or Kaf. However, the benefit of
69 // these letters as Non-Final letters outweighs the damage since these words
70 // are quite rare.
71}
72
98nsProbingState nsHebrewProber::HandleData(const char* aBuf, unsigned int aLen)
99{
100 // Both model probers say it's not them. No reason to continue.
101 if (GetState() == eNotMe)
102 return eNotMe;
103
104 const char *curPtr, *endPtr = aBuf+aLen;
105 char cur;
106
107 for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr)
108 {
109 cur = *curPtr;
110 if (cur == ' ') // We stand on a space - a word just ended
111 {
112 if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word
113 {
114 if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space]
115 ++mFinalCharLogicalScore;
116 else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space]
117 ++mFinalCharVisualScore;
118 }
119 }
120 else // Not standing on a space
121 {
122 if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space]
123 ++mFinalCharVisualScore;
124 }
125 mBeforePrev = mPrev;
126 mPrev = cur;
127 }
128
129 // Forever detecting, till the end or until both model probers return eNotMe (handled above).
130 return eDetecting;
131}
132
133// Make the decision: is it Logical or Visual?
134const char* nsHebrewProber::GetCharSetName()
135{
136 // If the final letter score distance is dominant enough, rely on it.
137 int finalsub = mFinalCharLogicalScore - mFinalCharVisualScore;
138 if (finalsub >= MIN_FINAL_CHAR_DISTANCE)
139 return LOGICAL_HEBREW_NAME;
140 if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE))
141 return VISUAL_HEBREW_NAME;
142
143 // It's not dominant enough, try to rely on the model scores instead.
144 float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence();
145 if (modelsub > MIN_MODEL_DISTANCE)
146 return LOGICAL_HEBREW_NAME;
147 if (modelsub < -(MIN_MODEL_DISTANCE))
148 return VISUAL_HEBREW_NAME;
149
150 // Still no good, back to final letter distance, maybe it'll save the day.
151 if (finalsub < 0)
152 return VISUAL_HEBREW_NAME;
153
154 // (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
155 return LOGICAL_HEBREW_NAME;
156}
157
158
159void nsHebrewProber::Reset(void)
160{
161 mFinalCharLogicalScore = 0;
162 mFinalCharVisualScore = 0;
163
164 // mPrev and mBeforePrev are initialized to space in order to simulate a word
165 // delimiter at the beginning of the data
166 mPrev = ' ';
167 mBeforePrev = ' ';
168}
169
170nsProbingState nsHebrewProber::GetState(void)
171{
172 // Remain active as long as any of the model probers are active.
173 if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe))
174 return eNotMe;
175 return eDetecting;
176}
177
178#ifdef DEBUG_PROBE
179void nsHebrewProber::DumpStatus()
180{
181 printf(" HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore);
182}
183#endif
184}
185
186
kencodingprober::nsCharSetProber::GetConfidence
virtual float GetConfidence(void)=0
kencodingprober::nsCharSetProber::GetState
virtual nsProbingState GetState(void)=0
kencodingprober::nsHebrewProber::GetCharSetName
virtual const char * GetCharSetName()
Definition: nsHebrewProber.cpp:134
kencodingprober::nsHebrewProber::GetState
virtual nsProbingState GetState(void)
Definition: nsHebrewProber.cpp:170
kencodingprober::nsHebrewProber::mPrev
char mPrev
Definition: nsHebrewProber.h:62
kencodingprober::nsHebrewProber::mLogicalProb
nsCharSetProber * mLogicalProb
Definition: nsHebrewProber.h:65
kencodingprober::nsHebrewProber::isNonFinal
static bool isNonFinal(char c)
Definition: nsHebrewProber.cpp:58
kencodingprober::nsHebrewProber::mVisualProb
nsCharSetProber * mVisualProb
Definition: nsHebrewProber.h:65
kencodingprober::nsHebrewProber::HandleData
virtual nsProbingState HandleData(const char *aBuf, unsigned int aLen)
HandleData Final letter analysis for logical-visual decision.
Definition: nsHebrewProber.cpp:98
kencodingprober::nsHebrewProber::mFinalCharLogicalScore
int mFinalCharLogicalScore
Definition: nsHebrewProber.h:59
kencodingprober::nsHebrewProber::mFinalCharVisualScore
int mFinalCharVisualScore
Definition: nsHebrewProber.h:59
kencodingprober::nsHebrewProber::isFinal
static bool isFinal(char c)
Definition: nsHebrewProber.cpp:53
kencodingprober::nsHebrewProber::Reset
virtual void Reset(void)
Definition: nsHebrewProber.cpp:159
kencodingprober::nsHebrewProber::mBeforePrev
char mBeforePrev
Definition: nsHebrewProber.h:62
kencodingprober
Definition: CharDistribution.cpp:37
kencodingprober::nsProbingState
nsProbingState
Definition: nsCharSetProber.h:34
kencodingprober::eNotMe
@ eNotMe
Definition: nsCharSetProber.h:37
kencodingprober::eDetecting
@ eDetecting
Definition: nsCharSetProber.h:35
MIN_MODEL_DISTANCE
#define MIN_MODEL_DISTANCE
Definition: nsHebrewProber.cpp:47
FINAL_TSADI
#define FINAL_TSADI
Definition: nsHebrewProber.cpp:38
NORMAL_PE
#define NORMAL_PE
Definition: nsHebrewProber.cpp:37
FINAL_NUN
#define FINAL_NUN
Definition: nsHebrewProber.cpp:34
VISUAL_HEBREW_NAME
#define VISUAL_HEBREW_NAME
Definition: nsHebrewProber.cpp:49
NORMAL_MEM
#define NORMAL_MEM
Definition: nsHebrewProber.cpp:33
NORMAL_NUN
#define NORMAL_NUN
Definition: nsHebrewProber.cpp:35
FINAL_MEM
#define FINAL_MEM
Definition: nsHebrewProber.cpp:32
FINAL_PE
#define FINAL_PE
Definition: nsHebrewProber.cpp:36
FINAL_KAF
#define FINAL_KAF
Definition: nsHebrewProber.cpp:30
MIN_FINAL_CHAR_DISTANCE
#define MIN_FINAL_CHAR_DISTANCE
Definition: nsHebrewProber.cpp:43
NORMAL_KAF
#define NORMAL_KAF
Definition: nsHebrewProber.cpp:31
LOGICAL_HEBREW_NAME
#define LOGICAL_HEBREW_NAME
Definition: nsHebrewProber.cpp:50
nsHebrewProber.h
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Mon Feb 20 2023 00:00:00 by doxygen 1.9.6 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KDECore

Skip menu "KDECore"
  • Main Page
  • Namespace List
  • Namespace Members
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Modules
  • Related Pages

kdelibs-4.14.38 API Reference

Skip menu "kdelibs-4.14.38 API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver
Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal