Skip to content

Commit ccdb674

Browse files
committed
✨ new I/O routines (must be included separately) to make printing easier!
- 🛠 fix improper size checking for a bunch of different ranges with new codepage-based stuff - 🛠 fix some copy-pasta/find-replace errors in licenses - 🛠 proper if constexpr branching to prevent falling off
1 parent 64810a3 commit ccdb674

File tree

11 files changed

+678
-25
lines changed

11 files changed

+678
-25
lines changed

CMakeLists.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,14 +246,16 @@ if (ZTD_CUNEICODE_EXAMPLES OR ZTD_CUNEICODE_SIMDUTF_SHARED)
246246
set(SIMDUTF_TOOLS OFF)
247247
set(SIMDUTF_SINGLE_HEADER OFF)
248248
set(SIMDUTF_TESTS OFF)
249-
set(BUILD_TESTING OFF)
249+
set(OLD_BUILD_TESTING ${BUILD_TESTING})
250+
set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
250251
FetchContent_Declare(
251252
simdutf
252253
GIT_REPOSITORY https://github.com/simdutf/simdutf
253254
GIT_SHALLOW ON
254255
GIT_TAG master
255256
EXCLUDE_FROM_ALL)
256257
FetchContent_MakeAvailable(simdutf)
258+
set(BUILD_TESTING ${OLD_BUILD_TESTING} CACHE BOOL "" FORCE)
257259
endfunction()
258260
simdutf_dependency_jail()
259261
add_subdirectory(shared)

documentation/source/conf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
rst_prolog = """
6969
.. |ub| replace:: ☢️☢️Undefined Behavior☢️☢️
7070
71-
.. |specializations_okay_different_types| replace:: User Specializations: ✔️ Okay! You can add other types to this classification by specializing the class template. Your specialization must have a type definition named ``type`` (as in, ``using type = ...;`` or `typedef ... type;``) inside of the class specialization that is ``public:``ly accessible. Note that specializing any type not explicitly marked with this notice is |ub|.
71+
.. |specializations_okay_different_types| replace:: User Specializations: ✔️ Okay! You can add other types to this classification by specializing the class template. Your specialization must have a type definition named ``type`` (as in, ``using type = ...;`` or ``typedef ... type;``) inside of the class specialization that is ``public:``ly accessible. Note that specializing any type not explicitly marked with this notice is |ub|.
7272
7373
.. |specializations_okay_true_false_type| replace:: User Specializations: ✔️ Okay! You can add other types to this classification by specializing the class template to a definition that derives from ``std::true_type``, or turn it off explicitly by having a definition that derives from ``std::false_type``. Note that specializing any type not explicitly marked with this notice is |ub|.
7474

examples/basic/source/registry_shift_jis_to_utf8.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,13 @@
3939
int main(int argc, char* argv[]) {
4040
(void)argc;
4141
(void)argv;
42+
4243
cnc_conversion_registry* registry = NULL;
4344
{
4445
cnc_open_err err
4546
= cnc_registry_new(&registry, cnc_registry_options_default);
4647
if (err != cnc_open_err_ok) {
47-
fprintf(stderr, "[error] could not open a new registry");
48+
fprintf(stderr, "[error] could not open a new registry conversion");
4849
return 1;
4950
}
5051
}
@@ -55,7 +56,7 @@ int main(int argc, char* argv[]) {
5556
cnc_open_err err = cnc_conv_new(
5657
registry, "shift-jis", "utf-8", &conversion, &conversion_info);
5758
if (err != cnc_open_err_ok) {
58-
fprintf(stderr, "[error] could not open a new registry");
59+
fprintf(stderr, "[error] could not open a new registry conversion");
5960
cnc_registry_delete(registry);
6061
return 2;
6162
}
Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
// ============================================================================
2+
//
3+
// ztd.cuneicode
4+
// Copyright © JeanHeyd "ThePhD" Meneide and Shepherd's Oasis, LLC
5+
// Contact: [email protected]
6+
//
7+
// Commercial License Usage
8+
// Licensees holding valid commercial ztd.cuneicode licenses may use this file
9+
// in accordance with the commercial license agreement provided with the
10+
// Software or, alternatively, in accordance with the terms contained in
11+
// a written agreement between you and Shepherd's Oasis, LLC.
12+
// For licensing terms and conditions see your agreement. For
13+
// further information contact [email protected].
14+
//
15+
// Apache License Version 2 Usage
16+
// Alternatively, this file may be used under the terms of Apache License
17+
// Version 2.0 (the "License"); you may not use this file except in compliance
18+
// with the License. You may obtain a copy of the License at
19+
//
20+
// https://www.apache.org/licenses/LICENSE-2.0
21+
//
22+
// Unless required by applicable law or agreed to in writing, software
23+
// distributed under the License is distributed on an "AS IS" BASIS,
24+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25+
// See the License for the specific language governing permissions and
26+
// limitations under the License.
27+
//
28+
// ========================================================================= //
29+
30+
#include <ztd/cuneicode.h>
31+
#include <ztd/cuneicode/io.h>
32+
33+
#include <ztd/idk/size.h>
34+
35+
#include <stdio.h>
36+
#include <stdint.h>
37+
#include <stdbool.h>
38+
#include <string.h>
39+
40+
#if ZTD_IS_ON(ZTD_PLATFORM_WINDOWS)
41+
42+
int main(int argc, char* argv[]) {
43+
(void)argc;
44+
(void)argv;
45+
46+
cnc_conversion_registry* registry = NULL;
47+
{
48+
cnc_open_err err
49+
= cnc_registry_new(&registry, cnc_registry_options_default);
50+
if (err != cnc_open_err_ok) {
51+
fprintf(stderr, "[error] could not open a new registry");
52+
return 1;
53+
}
54+
}
55+
56+
cnc_conversion* conversion = NULL;
57+
cnc_conversion_info conversion_info = { 0 };
58+
{
59+
cnc_open_err err = cnc_conv_new(registry, "utf-8",
60+
"win32-code_page-57007", &conversion, &conversion_info);
61+
if (err != cnc_open_err_ok) {
62+
fprintf(stderr, "[error] could not open a new registry conversion");
63+
cnc_registry_delete(registry);
64+
return 2;
65+
}
66+
}
67+
68+
cnc_conversion* conversion_reverse = NULL;
69+
cnc_conversion_info conversion_reverse_info = { 0 };
70+
{
71+
cnc_open_err err = cnc_conv_new(registry, "win32-code_page-57007",
72+
"utf-8", &conversion_reverse, &conversion_reverse_info);
73+
if (err != cnc_open_err_ok) {
74+
fprintf(stderr, "[error] could not open a new registry conversion");
75+
cnc_conv_delete(conversion);
76+
cnc_registry_delete(registry);
77+
return 3;
78+
}
79+
}
80+
81+
fprintf(stdout, "Opened a conversion from \"");
82+
fwrite(conversion_info.from_code_data,
83+
sizeof(*conversion_info.from_code_data), conversion_info.from_code_size,
84+
stdout);
85+
fprintf(stdout, "\" to \"");
86+
fwrite(conversion_info.to_code_data, sizeof(*conversion_info.to_code_data),
87+
conversion_info.to_code_size, stdout);
88+
if (conversion_info.is_indirect) {
89+
fprintf(stdout, "\" (through \"");
90+
fwrite(conversion_info.indirect_code_data,
91+
sizeof(*conversion_info.indirect_code_data),
92+
conversion_info.indirect_code_size, stdout);
93+
fprintf(stdout, "\")");
94+
}
95+
else {
96+
fprintf(stdout, "\"");
97+
}
98+
fprintf(stdout, "\n");
99+
100+
const char* const failed_conversion_result_title_str
101+
= "Conversion failed... \xF0\x9F\x98\xAD"; // UTF-8 bytes for 😭
102+
const char* const success_conversion_result_title_str
103+
= "Conversion succeeded \xF0\x9F\x8E\x89"; // UTF-8 bytes for 🎉
104+
const size_t success_conversion_result_title_str_size
105+
= strlen(success_conversion_result_title_str);
106+
const size_t failed_conversion_result_title_str_size
107+
= strlen(failed_conversion_result_title_str);
108+
109+
#if 0
110+
const ztd_char8_t input_data[]
111+
= u8"ସମସ୍ତ ମନୁଷ୍ୟ ଜନ୍ମକାଳରୁ ସ୍ଧୀନ ଏବଂ ମର୍ଯ୍ୟାଦା ଓ ଅଧିକାରରେ ସମାନ. ସେମାନଙ୍କଠାରେ ବୁଦ୍ଧି "
112+
u8"ଆଉ ବିବେକ ନିହିତ ଅଛି ଏବଂ ସେମାନଙ୍କୁ ପରସ୍ପର ପ୍ରତି ଭ୍ରାତୃତ୍ ମନୋଭାବରେ ବ୍ୟବହାର କରିବ ଉଚିତ୍";
113+
#else
114+
const ztd_char8_t input_data[] = u8"\0\0\0";
115+
#endif
116+
char intermediate_data[ztdc_c_array_size(input_data) * CNC_MC_MAX] = { 0 };
117+
const size_t starting_input_byte_size = sizeof(input_data);
118+
size_t input_byte_size = starting_input_byte_size;
119+
const unsigned char* input_bytes = (const unsigned char*)input_data;
120+
const size_t starting_intermediate_byte_size = sizeof(intermediate_data);
121+
size_t intermediate_byte_size = starting_intermediate_byte_size;
122+
unsigned char* intermediate_bytes = (unsigned char*)intermediate_data;
123+
cnc_mcerr intermediate_err = cnc_conv(conversion, &intermediate_byte_size,
124+
&intermediate_bytes, &input_byte_size, &input_bytes);
125+
const size_t input_bytes_read = starting_input_byte_size - input_byte_size;
126+
const size_t intermediate_bytes_written
127+
= starting_intermediate_byte_size - intermediate_byte_size;
128+
129+
if (intermediate_err != cnc_mcerr_ok) {
130+
// failed initial conversion
131+
// Use fwrite to prevent conversions / locale-sensitive-probing from
132+
// fprintf family of functions
133+
fprintf(stderr, "Intermediate conversion:\n");
134+
fwrite(failed_conversion_result_title_str,
135+
sizeof(*failed_conversion_result_title_str),
136+
failed_conversion_result_title_str_size, stderr);
137+
fprintf(stderr,
138+
"\n\tRead: %zu %zu-bit elements"
139+
"\n\tWrote: %zu %zu-bit elements\n\n",
140+
(size_t)(input_bytes_read / sizeof(*input_data)),
141+
(size_t)(sizeof(*input_data) * CHAR_BIT),
142+
(size_t)(intermediate_bytes_written),
143+
(size_t)(sizeof(*intermediate_data) * CHAR_BIT));
144+
145+
cnc_conv_delete(conversion);
146+
cnc_conv_delete(conversion_reverse);
147+
cnc_registry_delete(registry);
148+
return 5;
149+
}
150+
else {
151+
fprintf(stdout, "Intermediate conversion:\n");
152+
fwrite(success_conversion_result_title_str,
153+
sizeof(*success_conversion_result_title_str),
154+
success_conversion_result_title_str_size, stdout);
155+
fprintf(stdout,
156+
"\n\tRead: %zu %zu-bit elements"
157+
"\n\tWrote: %zu %zu-bit elements\n\n",
158+
(size_t)(input_bytes_read / sizeof(*input_data)),
159+
(size_t)(sizeof(*input_data) * CHAR_BIT),
160+
(size_t)(intermediate_bytes_written),
161+
(size_t)(sizeof(*intermediate_data) * CHAR_BIT));
162+
}
163+
164+
ztd_char8_t output_data[ztdc_c_array_size(input_data)] = { 0 };
165+
size_t intermediate_input_byte_size = intermediate_bytes_written;
166+
const unsigned char* intermediate_input_bytes
167+
= (const unsigned char*)intermediate_data;
168+
const size_t starting_output_byte_size = sizeof(output_data);
169+
size_t output_byte_size = starting_output_byte_size;
170+
unsigned char* output_bytes = (unsigned char*)output_data;
171+
cnc_mcerr err = cnc_conv(conversion_reverse, &output_byte_size, &output_bytes,
172+
&intermediate_input_byte_size, &intermediate_input_bytes);
173+
const size_t intermediate_bytes_read
174+
= intermediate_bytes_written - intermediate_input_byte_size;
175+
const size_t output_bytes_written
176+
= starting_output_byte_size - output_byte_size;
177+
if (err != cnc_mcerr_ok) {
178+
// failed back to the original
179+
fprintf(stderr, "Output conversion:\n");
180+
fwrite(failed_conversion_result_title_str,
181+
sizeof(*failed_conversion_result_title_str),
182+
failed_conversion_result_title_str_size, stderr);
183+
fprintf(stderr,
184+
"\n\tRead: %zu %zu-bit elements"
185+
"\n\tWrote: %zu %zu-bit elements\n\n",
186+
(size_t)(intermediate_bytes_read / sizeof(*intermediate_data)),
187+
(size_t)(sizeof(*intermediate_data) * CHAR_BIT),
188+
(size_t)(output_bytes_written),
189+
(size_t)(sizeof(*output_data) * CHAR_BIT));
190+
191+
cnc_conv_delete(conversion);
192+
cnc_conv_delete(conversion_reverse);
193+
cnc_registry_delete(registry);
194+
return 6;
195+
}
196+
else {
197+
fprintf(stdout, "Output conversion:\n");
198+
fwrite(success_conversion_result_title_str,
199+
sizeof(*success_conversion_result_title_str),
200+
success_conversion_result_title_str_size, stdout);
201+
fprintf(stdout,
202+
"\n\tRead: %zu %zu-bit elements"
203+
"\n\tWrote: %zu %zu-bit elements\n\n",
204+
(size_t)(intermediate_bytes_read / sizeof(*intermediate_data)),
205+
(size_t)(sizeof(*intermediate_data) * CHAR_BIT),
206+
(size_t)(output_bytes_written),
207+
(size_t)(sizeof(*output_data) * CHAR_BIT));
208+
}
209+
210+
fprintf(stdout, "Output conversion result:\n");
211+
cnc_fprint_str_c8n(stdout, ztdc_c_array_size(input_data), input_data);
212+
fprintf(stdout, "\nOriginal Input:\n");
213+
cnc_fprint_str_c8n(
214+
stdout, output_bytes_written / sizeof(*output_data), output_data);
215+
fprintf(stdout, "\n\n");
216+
217+
cnc_conv_delete(conversion);
218+
cnc_conv_delete(conversion_reverse);
219+
cnc_registry_delete(registry);
220+
return 0;
221+
}
222+
223+
#else
224+
225+
int main(int argc, char* argv[]) {
226+
// does not work on this platform
227+
(void)argc;
228+
(void)argv;
229+
return 0;
230+
}
231+
232+
#endif

examples/basic/source/windows_code_page.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
//
2828
// ========================================================================= //
2929

30-
#define _CRT_SECURE_NO_WARNINGS
31-
3230
#include <ztd/cuneicode.h>
3331

3432
#include <ztd/idk/size.h>
@@ -41,6 +39,9 @@
4139
#if ZTD_IS_ON(ZTD_PLATFORM_WINDOWS)
4240

4341
int main(int argc, char* argv[]) {
42+
(void)argc;
43+
(void)argv;
44+
4445
const char* const failed_conversion_result_title_str
4546
= "Conversion failed... \xF0\x9F\x98\xAD"; // UTF-8 bytes for 😭
4647
const char* const success_conversion_result_title_str
@@ -53,7 +54,7 @@ int main(int argc, char* argv[]) {
5354
(void)argv;
5455
const ztd_char32_t input_data[]
5556
= U"ସମସ୍ତ ମନୁଷ୍ୟ ଜନ୍ମକାଳରୁ ସ୍ଧୀନ ଏବଂ ମର୍ଯ୍ୟାଦା ଓ ଅଧିକାରରେ ସମାନ. ସେମାନଙ୍କଠାରେ ବୁଦ୍ଧି "
56-
U"ଆଉ ବିବେକ ନିହିତ ଅଛି ଏବଂ ସେମାନଙ୍କୁ ପରସ୍ପର ପ୍ରତି ଭ୍ରାତୃତ୍ ମନୋଭାବରେ ବ୍ୟବହାର କରିବା ଉଚିତ୍";
57+
U"ଆଉ ବିବେକ ନିହିତ ଅଛି ଏବଂ ସେମାନଙ୍କୁ ପରସ୍ପର ପ୍ରତି ଭ୍ରାତୃତ୍ ମନୋଭାବରେ ବ୍ୟବହାର କରିବ ଉଚିତ୍";
5758
const uint32_t win32_odia_code_page = 57007u;
5859
char intermediate_data[ztdc_c_array_size(input_data) * CNC_MC_MAX] = { 0 };
5960
cnc_mcstate_t intermediate_state = { 0 };

0 commit comments

Comments
 (0)