• Skip to content
  • Skip to link menu
  • KDE API Reference
  • kdelibs-4.14.38 API Reference
  • KDE Home
  • Contact Us
 

KDECore

  • kdecore
  • localization
  • probers
nsSBCharSetProber.h
Go to the documentation of this file.
1/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2/* -*- C++ -*-
3* Copyright (C) 1998 <developer@mozilla.org>
4*
5*
6* Permission is hereby granted, free of charge, to any person obtaining
7* a copy of this software and associated documentation files (the
8* "Software"), to deal in the Software without restriction, including
9* without limitation the rights to use, copy, modify, merge, publish,
10* distribute, sublicense, and/or sell copies of the Software, and to
11* permit persons to whom the Software is furnished to do so, subject to
12* the following conditions:
13*
14* The above copyright notice and this permission notice shall be included
15* in all copies or substantial portions of the Software.
16*
17* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24*/
25
26#ifndef NSSBCHARSETPROBER_H
27#define NSSBCHARSETPROBER_H
28
29#include "nsCharSetProber.h"
30
31#define SAMPLE_SIZE 64
32#define SB_ENOUGH_REL_THRESHOLD 1024
33#define POSITIVE_SHORTCUT_THRESHOLD (float)0.95
34#define NEGATIVE_SHORTCUT_THRESHOLD (float)0.05
35#define SYMBOL_CAT_ORDER 250
36#define NUMBER_OF_SEQ_CAT 4
37#define POSITIVE_CAT (NUMBER_OF_SEQ_CAT-1)
38#define NEGATIVE_CAT 0
39
40namespace kencodingprober {
41typedef struct
42{
43 const unsigned char *charToOrderMap; // [256] table use to find a char's order
44 const char *precedenceMatrix; // [SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency
45 float mTypicalPositiveRatio; // = freqSeqs / totalSeqs
46 bool keepEnglishLetter; // says if this script contains English characters (not implemented)
47 const char* charsetName;
48} SequenceModel;
49
50
51class KDE_NO_EXPORT nsSingleByteCharSetProber : public nsCharSetProber{
52public:
53 nsSingleByteCharSetProber(SequenceModel *model)
54 :mModel(model), mReversed(false), mNameProber(0) { Reset(); }
55 nsSingleByteCharSetProber(SequenceModel *model, bool reversed, nsCharSetProber* nameProber)
56 :mModel(model), mReversed(reversed), mNameProber(nameProber) { Reset(); }
57
58 virtual const char* GetCharSetName();
59 virtual nsProbingState HandleData(const char* aBuf, unsigned int aLen);
60 virtual nsProbingState GetState(void) {return mState;};
61 virtual void Reset(void);
62 virtual float GetConfidence(void);
63 virtual void SetOpion() {};
64
65 // This feature is not implemented yet. any current language model
66 // contain this parameter as false. No one is looking at this
67 // parameter or calling this method.
68 // Moreover, the nsSBCSGroupProber which calls the HandleData of this
69 // prober has a hard-coded call to FilterWithoutEnglishLetters which gets rid
70 // of the English letters.
71 bool KeepEnglishLetters() {return mModel->keepEnglishLetter;}; // (not implemented)
72
73#ifdef DEBUG_PROBE
74 virtual void DumpStatus();
75#endif
76
77protected:
78 nsProbingState mState;
79 const SequenceModel *mModel;
80 const bool mReversed; // true if we need to reverse every pair in the model lookup
81
82 //char order of last character
83 unsigned char mLastOrder;
84
85 unsigned int mTotalSeqs;
86 unsigned int mSeqCounters[NUMBER_OF_SEQ_CAT];
87
88 unsigned int mTotalChar;
89 //characters that fall in our sampling range
90 unsigned int mFreqChar;
91
92 // Optional auxiliary prober for name decision. created and destroyed by the GroupProber
93 nsCharSetProber* mNameProber;
94
95};
96
97
98extern SequenceModel Koi8rModel;
99extern SequenceModel Win1251Model;
100extern SequenceModel Latin5Model;
101extern SequenceModel MacCyrillicModel;
102extern SequenceModel Ibm866Model;
103extern SequenceModel Ibm855Model;
104extern SequenceModel Latin7Model;
105extern SequenceModel Win1253Model;
106extern SequenceModel Latin5BulgarianModel;
107extern SequenceModel Win1251BulgarianModel;
108extern SequenceModel Latin2HungarianModel;
109extern SequenceModel Win1250HungarianModel;
110extern SequenceModel Win1255Model;
111}
112#endif /* NSSBCHARSETPROBER_H */
113
kencodingprober::nsCharSetProber
Definition: nsCharSetProber.h:42
kencodingprober::nsSingleByteCharSetProber
Definition: nsSBCharSetProber.h:51
kencodingprober::nsSingleByteCharSetProber::nsSingleByteCharSetProber
nsSingleByteCharSetProber(SequenceModel *model, bool reversed, nsCharSetProber *nameProber)
Definition: nsSBCharSetProber.h:55
kencodingprober::nsSingleByteCharSetProber::KeepEnglishLetters
bool KeepEnglishLetters()
Definition: nsSBCharSetProber.h:71
kencodingprober::nsSingleByteCharSetProber::mModel
const SequenceModel * mModel
Definition: nsSBCharSetProber.h:79
kencodingprober::nsSingleByteCharSetProber::mTotalSeqs
unsigned int mTotalSeqs
Definition: nsSBCharSetProber.h:85
kencodingprober::nsSingleByteCharSetProber::mNameProber
nsCharSetProber * mNameProber
Definition: nsSBCharSetProber.h:93
kencodingprober::nsSingleByteCharSetProber::mLastOrder
unsigned char mLastOrder
Definition: nsSBCharSetProber.h:83
kencodingprober::nsSingleByteCharSetProber::GetState
virtual nsProbingState GetState(void)
Definition: nsSBCharSetProber.h:60
kencodingprober::nsSingleByteCharSetProber::SetOpion
virtual void SetOpion()
Definition: nsSBCharSetProber.h:63
kencodingprober::nsSingleByteCharSetProber::nsSingleByteCharSetProber
nsSingleByteCharSetProber(SequenceModel *model)
Definition: nsSBCharSetProber.h:53
kencodingprober::nsSingleByteCharSetProber::mTotalChar
unsigned int mTotalChar
Definition: nsSBCharSetProber.h:88
kencodingprober::nsSingleByteCharSetProber::mState
nsProbingState mState
Definition: nsSBCharSetProber.h:78
kencodingprober::nsSingleByteCharSetProber::mReversed
const bool mReversed
Definition: nsSBCharSetProber.h:80
kencodingprober::nsSingleByteCharSetProber::mFreqChar
unsigned int mFreqChar
Definition: nsSBCharSetProber.h:90
kencodingprober
Definition: CharDistribution.cpp:37
kencodingprober::Ibm855Model
SequenceModel Ibm855Model
Definition: LangCyrillicModel.cpp:336
kencodingprober::Win1251Model
SequenceModel Win1251Model
Definition: LangCyrillicModel.cpp:300
kencodingprober::Latin7Model
SequenceModel Latin7Model
Definition: LangGreekModel.cpp:217
kencodingprober::MacCyrillicModel
SequenceModel MacCyrillicModel
Definition: LangCyrillicModel.cpp:318
kencodingprober::nsProbingState
nsProbingState
Definition: nsCharSetProber.h:34
kencodingprober::Win1250HungarianModel
SequenceModel Win1250HungarianModel
Definition: LangHungarianModel.cpp:224
kencodingprober::Win1253Model
SequenceModel Win1253Model
Definition: LangGreekModel.cpp:226
kencodingprober::Latin2HungarianModel
SequenceModel Latin2HungarianModel
Definition: LangHungarianModel.cpp:215
kencodingprober::Latin5Model
SequenceModel Latin5Model
Definition: LangCyrillicModel.cpp:309
kencodingprober::Win1251BulgarianModel
SequenceModel KDE_NO_EXPORT Win1251BulgarianModel
Definition: LangBulgarianModel.cpp:227
kencodingprober::Win1255Model
SequenceModel Win1255Model
Definition: LangHebrewModel.cpp:198
kencodingprober::Koi8rModel
SequenceModel Koi8rModel
Definition: LangCyrillicModel.cpp:291
kencodingprober::Latin5BulgarianModel
SequenceModel KDE_NO_EXPORT Latin5BulgarianModel
Definition: LangBulgarianModel.cpp:218
kencodingprober::Ibm866Model
SequenceModel Ibm866Model
Definition: LangCyrillicModel.cpp:327
nsCharSetProber.h
NUMBER_OF_SEQ_CAT
#define NUMBER_OF_SEQ_CAT
Definition: nsSBCharSetProber.h:36
kencodingprober::SequenceModel
Definition: nsSBCharSetProber.h:42
kencodingprober::SequenceModel::mTypicalPositiveRatio
float mTypicalPositiveRatio
Definition: nsSBCharSetProber.h:45
kencodingprober::SequenceModel::charToOrderMap
const unsigned char * charToOrderMap
Definition: nsSBCharSetProber.h:43
kencodingprober::SequenceModel::keepEnglishLetter
bool keepEnglishLetter
Definition: nsSBCharSetProber.h:46
kencodingprober::SequenceModel::precedenceMatrix
const char * precedenceMatrix
Definition: nsSBCharSetProber.h:44
kencodingprober::SequenceModel::charsetName
const char * charsetName
Definition: nsSBCharSetProber.h:47
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Mon Feb 20 2023 00:00:00 by doxygen 1.9.6 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KDECore

Skip menu "KDECore"
  • Main Page
  • Namespace List
  • Namespace Members
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Modules
  • Related Pages

kdelibs-4.14.38 API Reference

Skip menu "kdelibs-4.14.38 API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver
Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal