在利用 hadoop 运行 MapReduce 项目时,提示报错(注意最后是Z):
Exception in thread "main" java.lang.UnsatisfiedLinkError:
org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
这个主要是因为本地启动运行,需要更改源代码中的对应配置。
在项目的根目录下(本项目为WordCount,根目录也就是java)新建包 org.apache.hadoop.io.nativeio ,添加类 NativeIO 即可。
package org.apache.hadoop.io.nativeio;/*** Licensed to the Apache Software Foundation (ASF) under one* or more contributor license agreements. See the NOTICE file* distributed with this work for additional information* regarding copyright ownership. The ASF licenses this file* to you under the Apache License, Version 2.0 (the* "License"); you may not use this file except in compliance* with the License. You may obtain a copy of the License at** http://www.apache.org/licenses/LICENSE-2.0** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;import sun.misc.Unsafe;import com.google.common.annotations.VisibleForTesting;/*** JNI wrappers for various native IO-related calls not available in Java. These* functions should generally be used alongside a fallback to another more* portable mechanism.*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class NativeIO {public static class POSIX {// Flags for open() call from bits/fcntl.hpublic static final int O_RDONLY = 00;public static final int O_WRONLY = 01;public static final int O_RDWR = 02;public static final int O_CREAT = 0100;public static final int O_EXCL = 0200;public static final int O_NOCTTY = 0400;public static final int O_TRUNC = 01000;public static final int O_APPEND = 02000;public static final int O_NONBLOCK = 04000;public static final int O_SYNC = 010000;public static final int O_ASYNC = 020000;public static final int O_FSYNC = O_SYNC;public static final int O_NDELAY = O_NONBLOCK;// Flags for posix_fadvise() from bits/fcntl.h/* No further special treatment. */public static final int POSIX_FADV_NORMAL = 0;/* Expect random page references. */public static final int POSIX_FADV_RANDOM = 1;/* Expect sequential page references. */public static final int POSIX_FADV_SEQUENTIAL = 2;/* Will need these pages. */public static final int POSIX_FADV_WILLNEED = 3;/* Don't need these pages. */public static final int POSIX_FADV_DONTNEED = 4;/* Data will be accessed once. */public static final int POSIX_FADV_NOREUSE = 5;/** Wait upon writeout of all pages in the range before performing the* write.*/public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;/** Initiate writeout of all those dirty pages in the range which are not* presently under writeback.*/public static final int SYNC_FILE_RANGE_WRITE = 2;/** Wait upon writeout of all pages in the range after performing the* write.*/public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;private static final Log LOG = LogFactory.getLog(NativeIO.class);private static boolean nativeLoaded = false;private static boolean fadvisePossible = true;private static boolean syncFileRangePossible = true;static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = "hadoop.workaround.non.threadsafe.getpwuid";static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = true;private static long cacheTimeout = -1;private static CacheManipulator cacheManipulator = new CacheManipulator();public static CacheManipulator getCacheManipulator() {return cacheManipulator;}public static void setCacheManipulator(CacheManipulator cacheManipulator) {POSIX.cacheManipulator = cacheManipulator;}/*** Used to manipulate the operating system cache.*/@VisibleForTestingpublic static class CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {POSIX.mlock(buffer, len);}public long getMemlockLimit() {return NativeIO.getMemlockLimit();}public long getOperatingSystemPageSize() {return NativeIO.getOperatingSystemPageSize();}public void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {POSIX.posixFadviseIfPossible(identifier, fd, offset, len, flags);}public boolean verifyCanMlock() {return NativeIO.isAvailable();}}/*** A CacheManipulator used for testing which does not actually call* mlock. This allows many tests to be run even when the operating* system does not allow mlock, or only allows limited mlocking.*/@VisibleForTestingpublic static class NoMlockCacheManipulator extends CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {LOG.info("mlocking " + identifier);}public long getMemlockLimit() {return 1125899906842624L;}public long getOperatingSystemPageSize() {return 4096;}public boolean verifyCanMlock() {return true;}}static {if (NativeCodeLoader.isNativeCodeLoaded()) {try {Configuration conf = new Configuration();workaroundNonThreadSafePasswdCalls = conf.getBoolean(WORKAROUND_NON_THREADSAFE_CALLS_KEY,WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);initNative();nativeLoaded = true;cacheTimeout = conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * 1000;LOG.debug("Initialized cache for IDs to User/Group mapping with a " + " cache timeout of "+ cacheTimeout / 1000 + " seconds.");} catch (Throwable t) {// This can happen if the user has an older version of// libhadoop.so// installed - in this case we can continue without native// IO// after warningLOG.error("Unable to initialize NativeIO libraries", t);}}}/*** Return true if the JNI-based native IO extensions are available.*/public static boolean isAvailable() {return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;}private static void assertCodeLoaded() throws IOException {if (!isAvailable()) {throw new IOException("NativeIO was not loaded");}}/** Wrapper around open(2) */public static native FileDescriptor open(String path, int flags, int mode) throws IOException;/** Wrapper around fstat(2) */private static native Stat fstat(FileDescriptor fd) throws IOException;/*** Native chmod implementation. On UNIX, it is a wrapper around chmod(2)*/private static native void chmodImpl(String path, int mode) throws IOException;public static void chmod(String path, int mode) throws IOException {if (!Shell.WINDOWS) {chmodImpl(path, mode);} else {try {chmodImpl(path, mode);} catch (NativeIOException nioe) {if (nioe.getErrorCode() == 3) {throw new NativeIOException("No such file or directory", Errno.ENOENT);} else {LOG.warn(String.format("NativeIO.chmod error (%d): %s", nioe.getErrorCode(), nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}}/** Wrapper around posix_fadvise(2) */static native void posix_fadvise(FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;/** Wrapper around sync_file_range(2) */static native void sync_file_range(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException;/*** Call posix_fadvise on the given file descriptor. See the manpage for* this syscall for more information. On systems where this call is not* available, does nothing.** @throws NativeIOException* if there is an error with the syscall*/static void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {if (nativeLoaded && fadvisePossible) {try {posix_fadvise(fd, offset, len, flags);} catch (UnsupportedOperationException uoe) {fadvisePossible = false;} catch (UnsatisfiedLinkError ule) {fadvisePossible = false;}}}/*** Call sync_file_range on the given file descriptor. See the manpage* for this syscall for more information. On systems where this call is* not available, does nothing.** @throws NativeIOException* if there is an error with the syscall*/public static void syncFileRangeIfPossible(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException {if (nativeLoaded && syncFileRangePossible) {try {sync_file_range(fd, offset, nbytes, flags);} catch (UnsupportedOperationException uoe) {syncFileRangePossible = false;} catch (UnsatisfiedLinkError ule) {syncFileRangePossible = false;}}}static native void mlock_native(ByteBuffer buffer, long len) throws NativeIOException;static native void munlock_native(ByteBuffer buffer, long len) throws NativeIOException;/*** Locks the provided direct ByteBuffer into memory, preventing it from* swapping out. After a buffer is locked, future accesses will not* incur a page fault.** See the mlock(2) man page for more information.** @throws NativeIOException*/static void mlock(ByteBuffer buffer, long len) throws IOException {assertCodeLoaded();if (!buffer.isDirect()) {throw new IOException("Cannot mlock a non-direct ByteBuffer");}mlock_native(buffer, len);}/*** Unlocks a locked direct ByteBuffer, allowing it to swap out of* memory. This is a no-op if the ByteBuffer was not previously locked.** See the munlock(2) man page for more information.** @throws NativeIOException*/public static void munlock(ByteBuffer buffer, long len) throws IOException {assertCodeLoaded();if (!buffer.isDirect()) {throw new IOException("Cannot munlock a non-direct ByteBuffer");}munlock_native(buffer, len);}/*** Unmaps the block from memory. See munmap(2).** There isn't any portable way to unmap a memory region in Java. So we* use the sun.nio method here. Note that unmapping a memory region* could cause crashes if code continues to reference the unmapped code.* However, if we don't manually unmap the memory, we are dependent on* the finalizer to do it, and we have no idea when the finalizer will* run.** @param buffer* The buffer to unmap.*/public static void munmap(MappedByteBuffer buffer) {if (buffer instanceof sun.nio.ch.DirectBuffer) {sun.misc.Cleaner cleaner = ((sun.nio.ch.DirectBuffer) buffer).cleaner();cleaner.clean();}}/** Linux only methods used for getOwner() implementation */private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException;private static native String getUserName(long uid) throws IOException;/*** Result type of the fstat call*/public static class Stat {private int ownerId, groupId;private String owner, group;private int mode;// Mode constantspublic static final int S_IFMT = 0170000; /* type of file */public static final int S_IFIFO = 0010000; /* named pipe (fifo) */public static final int S_IFCHR = 0020000; /* character special */public static final int S_IFDIR = 0040000; /* directory */public static final int S_IFBLK = 0060000; /* block special */public static final int S_IFREG = 0100000; /* regular */public static final int S_IFLNK = 0120000; /* symbolic link */public static final int S_IFSOCK = 0140000; /* socket */public static final int S_IFWHT = 0160000; /* whiteout */public static final int S_ISUID = 0004000; /** set user id on* execution*/public static final int S_ISGID = 0002000; /** set group id on* execution*/public static final int S_ISVTX = 0001000; /** save swapped text even* after use*/public static final int S_IRUSR = 0000400; /** read permission, owner*/public static final int S_IWUSR = 0000200; /** write permission,* owner*/public static final int S_IXUSR = 0000100; /** execute/search* permission, owner*/Stat(int ownerId, int groupId, int mode) {this.ownerId = ownerId;this.groupId = groupId;this.mode = mode;}Stat(String owner, String group, int mode) {if (!Shell.WINDOWS) {this.owner = owner;} else {this.owner = stripDomain(owner);}if (!Shell.WINDOWS) {this.group = group;} else {this.group = stripDomain(group);}this.mode = mode;}@Overridepublic String toString() {return "Stat(owner='" + owner + "', group='" + group + "'" + ", mode=" + mode + ")";}public String getOwner() {return owner;}public String getGroup() {return group;}public int getMode() {return mode;}}/*** Returns the file stat for a file descriptor.** @param fd* file descriptor.* @return the file descriptor file stat.* @throws IOException* thrown if there was an IO error while obtaining the file* stat.*/public static Stat getFstat(FileDescriptor fd) throws IOException {Stat stat = null;if (!Shell.WINDOWS) {stat = fstat(fd);stat.owner = getName(IdCache.USER, stat.ownerId);stat.group = getName(IdCache.GROUP, stat.groupId);} else {try {stat = fstat(fd);} catch (NativeIOException nioe) {if (nioe.getErrorCode() == 6) {throw new NativeIOException("The handle is invalid.", Errno.EBADF);} else {LOG.warn(String.format("NativeIO.getFstat error (%d): %s", nioe.getErrorCode(),nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}return stat;}private static String getName(IdCache domain, int id) throws IOException {Map
}