Skip to content

Commit 26c7604

Browse files
rfecherrfecher
committed
hbase mini cluster starts
1 parent a6dce75 commit 26c7604

File tree

6 files changed

+323
-71
lines changed

6 files changed

+323
-71
lines changed

‎core/store/src/main/java/org/locationtech/geowave/core/store/util/ClasspathUtils.java‎

Lines changed: 164 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,19 @@
1212
import java.io.FileFilter;
1313
import java.io.FileOutputStream;
1414
import java.io.IOException;
15+
import java.lang.management.ManagementFactory;
16+
import java.lang.reflect.Field;
17+
import java.lang.reflect.InvocationTargetException;
18+
import java.lang.reflect.Method;
1519
import java.net.URISyntaxException;
1620
import java.net.URL;
1721
import java.net.URLClassLoader;
1822
import java.util.ArrayList;
1923
import java.util.Collections;
2024
import java.util.Iterator;
2125
import java.util.List;
26+
import java.util.Collection;
27+
2228
import java.util.jar.Attributes;
2329
import java.util.jar.JarOutputStream;
2430
import java.util.jar.Manifest;
@@ -92,8 +98,76 @@ public static String setupPathingJarClassPath(
9298
return jarFile.getAbsolutePath();
9399
}
94100

101+
public static String setupPathingJarClassPath(
102+
final File dir,
103+
final Class context,
104+
final Collection<String> excludedJarPrefixes,
105+
final URL... additionalClasspathUrls) throws IOException {
106+
return setupPathingJarClassPath(
107+
new File(dir.getParentFile().getAbsolutePath() + File.separator + "pathing", "pathing.jar"),
108+
null,
109+
context,
110+
excludedJarPrefixes,
111+
additionalClasspathUrls);
112+
}
113+
114+
public static String setupPathingJarClassPath(
115+
final File jarFile,
116+
final String mainClass,
117+
final Class context,
118+
final Collection<String> excludedJarPrefixes,
119+
final URL... additionalClasspathUrls) throws IOException {
120+
121+
final File jarDir = jarFile.getParentFile();
122+
final String classpath = getClasspath(context, excludedJarPrefixes, additionalClasspathUrls);
123+
124+
if (!jarDir.exists()) {
125+
try {
126+
jarDir.mkdirs();
127+
} catch (final Exception e) {
128+
LOGGER.error("Failed to create pathing jar directory: " + e);
129+
return null;
130+
}
131+
}
132+
133+
if (jarFile.exists()) {
134+
try {
135+
jarFile.delete();
136+
} catch (final Exception e) {
137+
LOGGER.error("Failed to delete old pathing jar: " + e);
138+
return null;
139+
}
140+
}
141+
142+
// build jar
143+
final Manifest manifest = new Manifest();
144+
manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
145+
manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, classpath);
146+
if (mainClass != null) {
147+
manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, mainClass);
148+
}
149+
// HP Fortify "Improper Resource Shutdown or Release" false positive
150+
// target is inside try-as-resource clause (and is auto-closeable) and
151+
// the FileOutputStream
152+
// is closed implicitly by target.close()
153+
try (final JarOutputStream target =
154+
new JarOutputStream(new FileOutputStream(jarFile), manifest)) {
155+
156+
target.close();
157+
}
158+
159+
return jarFile.getAbsolutePath();
160+
}
161+
95162
private static String getClasspath(final Class context, final URL... additionalUrls)
96163
throws IOException {
164+
return getClasspath(context, (Collection<String>) null, additionalUrls);
165+
}
166+
167+
private static String getClasspath(
168+
final Class context,
169+
final Collection<String> excludedJarPrefixes,
170+
final URL... additionalUrls) throws IOException {
97171

98172
try {
99173
final ArrayList<ClassLoader> classloaders = new ArrayList<>();
@@ -109,49 +183,105 @@ private static String getClasspath(final Class context, final URL... additionalU
109183

110184
final StringBuilder classpathBuilder = new StringBuilder();
111185
for (final URL u : additionalUrls) {
112-
append(classpathBuilder, u);
186+
append(classpathBuilder, u, Collections.emptySet());
113187
}
114188

115-
// assume 0 is the system classloader and skip it
189+
boolean foundAnyUrls = false;
190+
191+
// Process each classloader
116192
for (int i = 0; i < classloaders.size(); i++) {
117193
final ClassLoader classLoader = classloaders.get(i);
118194

119195
if (classLoader instanceof URLClassLoader) {
196+
// Java 8 and custom URLClassLoaders (including HBaseMiniClusterClassLoader)
120197
for (final URL u : ((URLClassLoader) classLoader).getURLs()) {
121-
append(classpathBuilder, u);
198+
append(classpathBuilder, u, excludedJarPrefixes);
199+
foundAnyUrls = true;
122200
}
123201
} else if (classLoader instanceof VFSClassLoader) {
124202
final VFSClassLoader vcl = (VFSClassLoader) classLoader;
125203
for (final FileObject f : vcl.getFileObjects()) {
126-
append(classpathBuilder, f.getURL());
204+
append(classpathBuilder, f.getURL(), excludedJarPrefixes);
205+
foundAnyUrls = true;
127206
}
128207
} else {
129-
// Java 9+ classloaders (e.g., AppClassLoader/PlatformClassLoader) are not URLClassLoader.
130-
// Skip explicit URL extraction; we'll fall back to the system classpath if nothing was
131-
// added.
132-
continue;
208+
// Java 9+ classloaders (AppClassLoader, PlatformClassLoader)
209+
// Try to get URLs via reflection
210+
URL[] urls = null;
211+
212+
// Try the BuiltinClassLoader.ucp field (Java 9-16)
213+
try {
214+
final Field ucpField = classLoader.getClass().getDeclaredField("ucp");
215+
ucpField.setAccessible(true);
216+
final Object ucp = ucpField.get(classLoader);
217+
if (ucp != null) {
218+
final Method getURLsMethod = ucp.getClass().getMethod("getURLs");
219+
urls = (URL[]) getURLsMethod.invoke(ucp);
220+
}
221+
} catch (Exception e) {
222+
// Field/method not available, try alternative approach
223+
}
224+
225+
// Alternative: try getURLs() method directly (some custom loaders)
226+
if (urls == null) {
227+
try {
228+
final Method getUrlsMethod = classLoader.getClass().getMethod("getURLs");
229+
urls = (URL[]) getUrlsMethod.invoke(classLoader);
230+
} catch (Exception e) {
231+
// Method not available
232+
}
233+
}
234+
235+
if (urls != null) {
236+
for (final URL u : urls) {
237+
append(classpathBuilder, u, excludedJarPrefixes);
238+
foundAnyUrls = true;
239+
}
240+
}
133241
}
134242
}
135243

136-
if (classpathBuilder.length() == 0) {
244+
// If we didn't find any URLs from classloaders, fall back to system classpath
245+
if (!foundAnyUrls || classpathBuilder.length() == 0) {
137246
final String sysCp = System.getProperty("java.class.path");
138-
return (sysCp == null) ? "" : sysCp;
247+
if (sysCp != null && !sysCp.isEmpty()) {
248+
final String[] parts = sysCp.split(java.io.File.pathSeparator);
249+
for (final String part : parts) {
250+
if (part == null || part.isEmpty())
251+
continue;
252+
try {
253+
append(classpathBuilder, new java.io.File(part).toURI().toURL(), excludedJarPrefixes);
254+
} catch (java.net.MalformedURLException e) {
255+
// skip invalid entry
256+
}
257+
}
258+
}
139259
}
140-
if (classpathBuilder.charAt(0) == ' ') {
260+
261+
// Remove leading space if present
262+
if (classpathBuilder.length() > 0 && classpathBuilder.charAt(0) == ' ') {
141263
classpathBuilder.deleteCharAt(0);
142264
}
265+
143266
return classpathBuilder.toString();
144267

145268
} catch (final URISyntaxException e) {
146269
throw new IOException(e);
147270
}
148271
}
149272

150-
private static void append(final StringBuilder classpathBuilder, final URL url)
151-
throws URISyntaxException {
273+
private static void append(
274+
final StringBuilder classpathBuilder,
275+
final URL url,
276+
final Collection<String> excludedJarPrefixes) throws URISyntaxException {
152277

153278
final File file = new File(url.toURI());
154279

280+
// Skip excluded JARs by filename prefix
281+
if (isExcludedJar(file, excludedJarPrefixes)) {
282+
return;
283+
}
284+
155285
// do not include dirs containing hadoop or accumulo site files
156286
if (!containsSiteFile(file)) {
157287
final int index = file.getAbsolutePath().indexOf(":\\");
@@ -173,6 +303,27 @@ private static void append(final StringBuilder classpathBuilder, final URL url)
173303
}
174304
}
175305

306+
private static boolean isExcludedJar(
307+
final File file,
308+
final Collection<String> excludedJarPrefixes) {
309+
if (excludedJarPrefixes == null || excludedJarPrefixes.isEmpty()) {
310+
return false;
311+
}
312+
if (!file.isFile()) {
313+
return false;
314+
}
315+
final String name = file.getName();
316+
if (!name.endsWith(".jar")) {
317+
return false;
318+
}
319+
for (final String prefix : excludedJarPrefixes) {
320+
if (prefix != null && !prefix.isEmpty() && name.startsWith(prefix)) {
321+
return true;
322+
}
323+
}
324+
return false;
325+
}
326+
176327
private static boolean containsSiteFile(final File f) {
177328
if (f.isDirectory()) {
178329
final File[] sitefile = f.listFiles(new FileFilter() {

‎extensions/cli/hbase-embed/src/main/java/org/locationtech/geowave/datastore/hbase/cli/GeoWaveHBaseUtility.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.apache.hadoop.fs.Path;
1515
import org.apache.hadoop.fs.permission.FsPermission;
1616
import org.apache.hadoop.hbase.HBaseTestingUtility;
17-
import org.apache.hadoop.hbase.MiniHBaseCluster;
1817

1918
public class GeoWaveHBaseUtility extends HBaseTestingUtility {
2019

‎extensions/cli/hbase-embed/src/main/java/org/locationtech/geowave/datastore/hbase/cli/HBaseMiniCluster.java‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import java.security.PrivilegedExceptionAction;
1313
import java.util.List;
1414
import org.apache.hadoop.conf.Configuration;
15-
import org.apache.hadoop.hbase.HBaseConfiguration;
1615
import org.apache.hadoop.hbase.client.Connection;
1716
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
1817
import org.apache.hadoop.hbase.security.User;
@@ -34,7 +33,7 @@ public class HBaseMiniCluster {
3433

3534
private final String zkDataDir;
3635

37-
private GeoWaveHBaseUtility hbaseLocalCluster;
36+
private Object hbaseLocalCluster;
3837
private final String hbaseLibDir;
3938
private final String hbaseDataDir;
4039
private final int numRegionServers;
@@ -72,11 +71,15 @@ public void setup() {
7271
}
7372

7473
final ClassLoader prevCl = Thread.currentThread().getContextClassLoader();
75-
final ClassLoader hbaseMiniClusterCl = prevCl;
76-
// HBaseMiniClusterClassLoader.getInstance(prevCl, hbaseLibDir);
74+
final ClassLoader hbaseMiniClusterCl =
75+
HBaseMiniClusterClassLoader.getInstance(prevCl, hbaseLibDir);
7776
Thread.currentThread().setContextClassLoader(hbaseMiniClusterCl);
7877
try {
79-
final Configuration conf = HBaseConfiguration.create();
78+
final Configuration conf =
79+
(Configuration) Class.forName(
80+
"org.apache.hadoop.hbase.HBaseConfiguration",
81+
true,
82+
hbaseMiniClusterCl).getMethod("create").invoke(null);
8083
System.setProperty("test.build.data.basedirectory", hbaseDataDir);
8184
conf.setBoolean("hbase.online.schema.update.enable", true);
8285
conf.setBoolean("hbase.defaults.for.version.skip", true);
@@ -113,9 +116,18 @@ public void setup() {
113116

114117
// HBaseTestingUtility must be loaded dynamically by the
115118
// minicluster class loader
116-
hbaseLocalCluster = new GeoWaveHBaseUtility(conf);
119+
hbaseLocalCluster =
120+
Class.forName(
121+
// "org.apache.hadoop.hbase.HBaseTestingUtility",
122+
"org.locationtech.geowave.datastore.hbase.cli.GeoWaveHBaseUtility",
123+
true,
124+
hbaseMiniClusterCl).getConstructor(Configuration.class).newInstance(conf);
125+
117126
// Start the cluster
118-
hbaseLocalCluster.startMiniCluster(1, numRegionServers);
127+
hbaseLocalCluster.getClass().getMethod(
128+
"startMiniHBaseCluster",
129+
Integer.TYPE,
130+
Integer.TYPE).invoke(hbaseLocalCluster, 1, numRegionServers);
119131

120132

121133
if (enableVisibility) {

0 commit comments

Comments
 (0)