• Skip to content
  • Skip to link menu
  • KDE API Reference
  • kdelibs-4.14.38 API Reference
  • KDE Home
  • Contact Us
 

KDECore

  • kdecore
  • localization
  • probers
nsCharSetProber.cpp
Go to the documentation of this file.
1/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2/* -*- C++ -*-
3* Copyright (C) 1998 <developer@mozilla.org>
4*
5*
6* Permission is hereby granted, free of charge, to any person obtaining
7* a copy of this software and associated documentation files (the
8* "Software"), to deal in the Software without restriction, including
9* without limitation the rights to use, copy, modify, merge, publish,
10* distribute, sublicense, and/or sell copies of the Software, and to
11* permit persons to whom the Software is furnished to do so, subject to
12* the following conditions:
13*
14* The above copyright notice and this permission notice shall be included
15* in all copies or substantial portions of the Software.
16*
17* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24*/
25
26#include "nsCharSetProber.h"
27
28#include <stdlib.h>
29
30namespace kencodingprober {
31//This filter applies to all scripts which do not use English characters
32bool nsCharSetProber::FilterWithoutEnglishLetters(const char* aBuf, unsigned int aLen, char** newBuf, unsigned int& newLen)
33{
34 char *newptr;
35 char *prevPtr, *curPtr;
36
37 bool meetMSB = false;
38 newptr = *newBuf = (char*)malloc(aLen);
39 if (!newptr)
40 return false;
41
42 for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; ++curPtr)
43 {
44 if (*curPtr & 0x80)
45 {
46 meetMSB = true;
47 }
48 else if (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z')
49 {
50 //current char is a symbol, most likely a punctuation. we treat it as segment delimiter
51 if (meetMSB && curPtr > prevPtr)
52 //this segment contains more than single symbol, and it has upper ASCII, we need to keep it
53 {
54 while (prevPtr < curPtr) *newptr++ = *prevPtr++;
55 prevPtr++;
56 *newptr++ = ' ';
57 meetMSB = false;
58 }
59 else //ignore current segment. (either because it is just a symbol or just an English word)
60 prevPtr = curPtr+1;
61 }
62 }
63 if (meetMSB && curPtr > prevPtr)
64 while (prevPtr < curPtr) *newptr++ = *prevPtr++;
65
66 newLen = newptr - *newBuf;
67
68 return true;
69}
70
71//This filter applies to all scripts which contain both English characters and upper ASCII characters.
72bool nsCharSetProber::FilterWithEnglishLetters(const char* aBuf, unsigned int aLen, char** newBuf, unsigned int& newLen)
73{
74 //do filtering to reduce load to probers
75 char *newptr;
76 char *prevPtr, *curPtr;
77 bool isInTag = false;
78
79 newptr = *newBuf = (char*)malloc(aLen);
80 if (!newptr)
81 return false;
82
83 for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; ++curPtr)
84 {
85 if (*curPtr == '>')
86 isInTag = false;
87 else if (*curPtr == '<')
88 isInTag = true;
89
90 if (!(*curPtr & 0x80) &&
91 (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') )
92 {
93 if (curPtr > prevPtr && !isInTag) // Current segment contains more than just a symbol
94 // and it is not inside a tag, keep it.
95 {
96 while (prevPtr < curPtr) *newptr++ = *prevPtr++;
97 prevPtr++;
98 *newptr++ = ' ';
99 }
100 else
101 prevPtr = curPtr+1;
102 }
103 }
104
105 // If the current segment contains more than just a symbol
106 // and it is not inside a tag then keep it.
107 if (!isInTag)
108 while (prevPtr < curPtr)
109 *newptr++ = *prevPtr++;
110
111 newLen = newptr - *newBuf;
112
113 return true;
114}
115}
116
117
kencodingprober::nsCharSetProber::FilterWithoutEnglishLetters
static bool FilterWithoutEnglishLetters(const char *aBuf, unsigned int aLen, char **newBuf, unsigned int &newLen)
Definition: nsCharSetProber.cpp:32
kencodingprober::nsCharSetProber::FilterWithEnglishLetters
static bool FilterWithEnglishLetters(const char *aBuf, unsigned int aLen, char **newBuf, unsigned int &newLen)
Definition: nsCharSetProber.cpp:72
kencodingprober
Definition: CharDistribution.cpp:37
nsCharSetProber.h
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Mon Feb 20 2023 00:00:00 by doxygen 1.9.6 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KDECore

Skip menu "KDECore"
  • Main Page
  • Namespace List
  • Namespace Members
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Modules
  • Related Pages

kdelibs-4.14.38 API Reference

Skip menu "kdelibs-4.14.38 API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver
Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal